This is an automated email from the ASF dual-hosted git repository.

tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new 1c2a4d7f3b TIKA-5027: junk utf16 misreads (#3602)
1c2a4d7f3b is described below

commit 1c2a4d7f3b6534c9ce440b45cf6ae8e79709b4fc
Author: Tim Allison <[email protected]>
AuthorDate: Thu Nov 12 16:52:56 2026 -0500

    TIKA-5027: junk utf16 misreads (#3602)
    
    * next round of fixes
    
    * TIKA-5052: retrain the charset model with C0-control negatives; arbitrate 
lone UTF-16 picks
    
    Co-Authored-By: Claude Fable 5.1 <[email protected]>
---
 CHANGES.txt                                        |  16 +++
 .../tika/detect/html/HtmlEncodingDetector.java     |  11 ++
 .../tika/parser/html/HtmlEncodingDetectorTest.java |  16 +--
 .../ml/chardetect/MojibusterEncodingDetector.java  |  25 +++++
 .../org/apache/tika/ml/chardetect/hashed-ngram.hng | Bin 2742509 -> 2742509 
bytes
 .../tika/ml/chardetect/hashed-ngram.hng.provenance |  10 +-
 .../ml/chardetect/WideUnicodeDetectionTest.java    |   6 +-
 .../tika/ml/chardetect/hashed-ngram-parity.tsv     |  24 ++--
 .../tika/ml/junkdetect/tools/JunkFamilies.java     |  26 +++++
 .../tika-ml-junkdetect-tools/train-junk-model.sh   |   2 +-
 .../org/apache/tika/ml/junkdetect/junkdetect.bin   | Bin 3058253 -> 3050818 
bytes
 .../tika/ml/junkdetect/junkdetect.bin.provenance   |  52 ++++-----
 .../tika/ml/junkdetect/HashedArbitrationTest.java  |  31 ++++--
 .../ml/junkdetect/LoneWideArbitrationTest.java     | 124 +++++++++++++++++++++
 .../parser/html/HtmlEncodingDetectionTest.java     |  61 ++++++++++
 15 files changed, 345 insertions(+), 59 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index e524a8437b..40b5cf9457 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -410,6 +410,11 @@ Release 4.3.0 - ???
      is honored only when the bytes after it decode in that charset, and a 
rejected BOM
      is dropped rather than left in the text as junk characters (TIKA-5019).
 
+   * An HTML meta charset naming UTF-16 (or UTF-32) is read as UTF-8, as the 
WHATWG
+     prescan does: a meta readable as ASCII means the bytes are not UTF-16. A 
page with a
+     UTF-8 BOM and a UTF-16BE meta was decoded as UTF-16BE, turning its text 
into CJK
+     characters (TIKA-5019).
+
    * Charset detection reads UTF-8 with sparse corruption (up to one bad 
sequence per
      ten valid ones, existing U+FFFD not counted) as UTF-8 (TIKA-5019).
 
@@ -448,6 +453,17 @@ Release 4.3.0 - ???
      UTF-32 code points: binary with a few NULs in its header (a shader file 
of ASCII 0/1
      after an 80 37 00 00 header) was reported as UTF-32BE (TIKA-5052).
 
+   * Charset detection no longer reads ASCII text with a few control 
characters (logs with
+     escape or form-feed bytes, IRC colour codes, record separators, banners) 
as BOM-less
+     UTF-16: the model was retrained with such text, and when it still returns 
UTF-16 or
+     UTF-32 as its only candidate, the junk filter now compares that decode 
against
+     windows-1252 and keeps whichever reads as text (TIKA-5052).
+
+   * The junk model that arbitrates between charset candidates was retrained 
with UTF-16
+     misreads of every script (text read as UTF-16, and UTF-16 text read as a 
single-byte
+     charset) and with fully vocalized Arabic, so a BOM-less UTF-16 decode of 
vocalized
+     Arabic is no longer scored below its windows-1252 misread (TIKA-5027).
+
    * XHTML output no longer contains empty or whitespace-only 
b/strong/i/em/sup/sub
      elements, and whitespace at their edges is written outside them (<b>Note: 
</b>
      becomes <b>Note:</b> ); u and s are unchanged. The Markdown writer does 
the same for
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/detect/html/HtmlEncodingDetector.java
 
b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/detect/html/HtmlEncodingDetector.java
index b9f025f477..9083a6da2b 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/detect/html/HtmlEncodingDetector.java
+++ 
b/tika-encoding-detectors/tika-encoding-detector-html/src/main/java/org/apache/tika/detect/html/HtmlEncodingDetector.java
@@ -181,6 +181,7 @@ public class HtmlEncodingDetector implements 
EncodingDetector {
         if (charset == null) {
             return Collections.emptyList();
         }
+        charset = asciiReadableCharset(charset);
         return List.of(new EncodingResult(charset, 1.0f, charset.name(),
                 EncodingResult.ResultType.DECLARATIVE));
     }
@@ -218,6 +219,16 @@ public class HtmlEncodingDetector implements 
EncodingDetector {
         return false;
     }
 
+    /**
+     * WHATWG prescan: a meta naming UTF-16 means UTF-8. The meta was read as 
ASCII, so the
+     * bytes cannot be UTF-16 (or UTF-32); a BOM or the byte statistics still 
speak for those.
+     */
+    static Charset asciiReadableCharset(Charset charset) {
+        String name = charset.name();
+        return name.startsWith("UTF-16") || name.startsWith("UTF-32")
+                ? StandardCharsets.UTF_8 : charset;
+    }
+
     //returns null if no charset was found
     private Charset findCharset(String s) {
 
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
 
b/tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
index d6c3161faf..82a1c93174 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
+++ 
b/tika-encoding-detectors/tika-encoding-detector-html/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectorTest.java
@@ -38,12 +38,14 @@ public class HtmlEncodingDetectorTest {
         assertWindows1252("<meta charset='WINDOWS-1252'>");
     }
 
+    /** A meta read as ASCII rules out UTF-16 and UTF-32 bytes: WHATWG reads 
it as UTF-8. */
     @Test
-    @Disabled("can we can prove this harms detection")
     public void utf16() throws IOException {
-        // According to the specification 'If charset is a UTF-16 encoding,
-        // then set charset to UTF-8.'
-        assertCharset("<meta charset='UTF-16BE'>", StandardCharsets.UTF_8);
+        for (String label : new String[]{"UTF-16BE", "utf-16le", "UTF-32", 
"utf-32be"}) {
+            assertCharset("<meta charset='" + label + "'>", 
StandardCharsets.UTF_8);
+        }
+        assertCharset("<meta http-equiv='content-type' content='text/html; 
charset=UTF-16LE'>",
+                StandardCharsets.UTF_8);
     }
 
     @Test
@@ -93,9 +95,9 @@ public class HtmlEncodingDetectorTest {
     }
 
     @Test
-    public void nakedUtf16IsUtf16Le() throws IOException {
-        // WHATWG: naked 'utf-16' (no BOM) defaults to UTF-16LE.
-        assertCharset("<meta charset='utf-16'>", StandardCharsets.UTF_16LE);
+    public void nakedUtf16IsUtf8() throws IOException {
+        // WHATWG: the label 'utf-16' is UTF-16LE, which the prescan then 
reads as UTF-8.
+        assertCharset("<meta charset='utf-16'>", StandardCharsets.UTF_8);
     }
 
     @Test
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index c829b0197c..f67258b541 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++ 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -135,6 +135,10 @@ public class MojibusterEncodingDetector implements 
EncodingDetector {
     /** Confidence for the windows-1252 fallback emitted on empty/ASCII 
probes: an "I don't know". */
     private static final float FALLBACK_CONFIDENCE = 0.1f;
 
+    /** Confidence of the windows-1252 alternative added behind a lone 
statistical UTF-16/32 pick.
+     *  Kept at JunkFilter's tail band (0.5) so it survives when both 
endiannesses are listed. */
+    private static final float LONE_WIDE_ALTERNATIVE_CONF = 0.5f;
+
     /**
      * Maximum fraction of malformed-UTF-8 bytes we tolerate before
      * disqualifying the model's UTF-8 pick.  Real-world UTF-8 files often 
contain
@@ -402,6 +406,18 @@ public class MojibusterEncodingDetector implements 
EncodingDetector {
         // JunkFilter can arbitrate within-family ambiguities (e.g. GB18030 vs
         // x-windows-949), using its language-quality scoring.
         EncodingResult top = finalResults.get(0);
+        // A statistical UTF-16/32 pick with no single-byte alternative gives 
JunkFilter nothing to
+        // weigh it against, and a wrong wide pick turns readable text into 
ideographs.  Offer
+        // windows-1252 as the alternative decode (7-bit bytes decode the same 
under every ASCII
+        // superset); JunkFilter's text-quality model then decides between the 
two decodes.
+        if (top.getResultType() == EncodingResult.ResultType.STATISTICAL
+                && isWideUnicode(top.getCharset()) && allWide(finalResults)) {
+            finalResults = new java.util.ArrayList<>(finalResults);
+            finalResults.add(new EncodingResult(Charset.forName(WIN1252), 
LONE_WIDE_ALTERNATIVE_CONF,
+                    WIN1252, EncodingResult.ResultType.STATISTICAL));
+            LOG.trace("mojibuster lone wide pick {}: windows-1252 added as the 
alternative decode",
+                    top.getCharset().name());
+        }
         List<EncodingResult> toReturn = (top.getResultType() == 
EncodingResult.ResultType.STRUCTURAL)
                 ? List.of(top) : finalResults;
         if (LOG.isTraceEnabled()) {
@@ -505,6 +521,15 @@ public class MojibusterEncodingDetector implements 
EncodingDetector {
         return true;
     }
 
+    private static boolean allWide(List<EncodingResult> results) {
+        for (EncodingResult r : results) {
+            if (!isWideUnicode(r.getCharset())) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     private static boolean isWideUnicode(Charset cs) {
         String name = cs.name();
         return name.startsWith("UTF-16") || name.startsWith("UTF-32");
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
index 4c2617d6aa..b4ab5c2056 100644
Binary files 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
 and 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
 differ
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
index b4f63e32e9..706b09e320 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
+++ 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
@@ -13,11 +13,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-model_sha256   b1a64c01b6404c1dbe9644a054907e4975e9cb527b12c31895612806f72ebeb4
-model_md5      7cd1606fdff904d026678e966c18dab6
+model_sha256   056fcf927453a6eacdcd4f7e463a6d58e8862184990038f62f8dbe6738e5455e
+model_md5      a536ff669cd1610cb9212774d72ab2a9
 model_bytes    2742509
 format HNG2 (see tika-ml/tika-ml-chardetect/README.md)
-status datafix "ALL", seed 7, 2026-11-07; trained on gold v1.12 (R-s11 recipe 
plus the data fixes below)
+status utf16neg "F", seed 13, 2026-11-11; datafix "ALL" (2026-11-07) plus 
C0-control negatives; gold v1.12
 
 Architecture
   43 classes: 37 legacy/UTF-8 charsets (IBM420/IBM424 as -ltr/-rtl pairs), 
IBM437, binary,
@@ -38,6 +38,10 @@ Training data (nothing from the 1.2M regression corpus)
   Vocalized Arabic in windows-1256 / ISO-8859-6; 
Spanish/Portuguese/Catalan/Galician Wikisource and Spanish
   Wikiquote in windows-1252 (dialogue/literary text with ¡ ¿ º ª); Kurmanji 
Wikipedia in windows-1254.
   Text with sparse single NULs (<= 2%) labelled by its charset.
+  C0 controls: 3% of single-byte/UTF-8 rows (not EBCDIC) get all C0 types 
(00-1F except TAB/LF/CR) at
+  weights from per-byte document counts (gold v1.12 web train + a 1.2M-corpus 
sample; counts only, no
+  bytes; floor 5), rate 1e-4..1e-2, inserted only between 7-bit bytes, half of 
them with high bytes
+  removed; labelled by their charset.
   Tier B rows whose label set decodes identically take windows-1252 as a hard 
target.
   Negatives: generated UTF-32LE/BE rows (own classes); generated binary (NUL 
records, deflate, bytecode,
   integer arrays, random, executables; the scattered-NUL text family was 
removed) plus the gold v1.12 fit binary-verdict documents
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
index 4c21438acb..28eb750f15 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
+++ 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/WideUnicodeDetectionTest.java
@@ -196,7 +196,11 @@ public class WideUnicodeDetectionTest {
         WideUnicodeDetector.Result wide = WideUnicodeDetector.analyze(bytes);
         assertTrue(wide.invalidUtf32Be);
         assertTrue(wide.invalidUtf32Le);
-        assertEquals("windows-1252", top(bytes));
+        String top = top(bytes);
+        assertFalse(top.startsWith("UTF-32") || top.startsWith("UTF-16"), top);
+        // which Latin label the model picks is seed-dependent; its decode 
must equal windows-1252's
+        assertEquals(new String(bytes, Charset.forName("windows-1252")),
+                new String(bytes, Charset.forName(top)));
     }
 
     @ParameterizedTest
diff --git 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
index f0a107191a..ad3f576b79 100644
--- 
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
+++ 
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
@@ -14,16 +14,16 @@
 #  limitations under the License.
 # Parity fixture for HashedNgramModelParityTest: probe name, probe bytes 
(hex), reference logits.
 # Regenerate with the reference scorer whenever the model changes.
-model-md5      7cd1606fdff904d026678e966c18dab6
+model-md5      a536ff669cd1610cb9212774d72ab2a9
 ascii-abstain  
506c61696e204153434949206f6e6c792c206e6f7468696e6720746f2073636f72652e0d0a      
abstain
-windows-1252   
4c6520636166e920657374207472e87320626f6e2c206d657263692062656175636f757020706f757220766f747265206163637565696c20e02050617269732e
        
-6.26243,-8.276049,-7.5950513,-15.685164,-18.508417,-18.136816,-11.677787,-11.436772,-11.429723,-1.6022514,-0.6278504,-2.9343238,-3.2420566,4.612255,-1.5458668,3.170168,-2.1325564,0.52321064,-1.605255,-4.855722,-6.62645,-14.765057,-5.6225595,-1.4943169,-2.3204155,7.4401264,-2.5516288,6.9778447,-0.6250175,-4.572941,-0.37654838,1.202957,-0.26864636,-
 [...]
-UTF-8  
4772c3bcc39f6520617573204dc3bc6e6368656e20e2809420736368c3b66e65204772c3bcc39f652c20e697a5e69cace8aa9e2e
        
-0.5827861,-6.6113305,0.045845836,-9.239936,-5.9060984,-5.5219054,-9.469475,-9.633721,-8.441814,-7.54745,-13.777663,-4.715149,-3.4075067,-15.77235,-16.508482,-18.398487,-12.1268635,-15.299498,-14.69563,0.13925156,-9.128676,-5.489176,14.18173,-10.594682,-4.7317257,-15.118749,-12.118952,-22.418028,-21.3163,-10.658545,-15.678664,-10.982281,-4.5749016,-1.7791243,-8.110289,-4.906046,
 [...]
-Shift_JIS      
93fa967b8cea82cc8365834c8358836782c582b7814282b182ea82cd83658358836782c582b78142
        
-7.482871,-6.7766385,4.062569,3.0263946,-7.778361,-7.413558,-3.5662513,-3.9060385,0.2243073,-13.015116,-18.522057,-14.260608,-3.560188,-18.54181,-17.02574,-40.14331,-13.6332655,-19.063444,-9.753614,-0.43981814,-7.344433,20.272371,-3.1040049,-1.7563919,-3.9684286,-15.076569,4.173208,-24.263617,-6.0777817,-9.948965,-29.124353,-13.09328,0.19826484,2.1880326,-17.013842,-5.945684,-19.390018,-12.687593,4
 [...]
-ISO-8859-7     b6f1e8f1e120e3e9e120f4e7ed20f0f1fcebe7f8e720f4e7f220e2dfe1f22e  
-0.8203472,-7.461944,-1.9833272,-2.4042392,-7.2492504,-7.193231,-8.779393,-9.237593,1.1447276,-15.360343,-27.163002,-3.4032931,-5.120804,-11.597296,-6.46171,-13.295239,-2.9843912,-4.196037,16.045206,-3.2323692,0.98040044,-2.3271854,-10.323003,-10.868758,-4.4015923,-11.062085,10.325654,-13.378147,-3.8846247,-4.175025,-19.496069,-7.166667,-6.739304,0.17751911,-13.286078,-5.898166,-4.800658,-19.094826,-2.1375165,-3.95
 [...]
-windows-1251   
cdeee2e0ff20eff0eee3f0e0ecece020efee20eff0eef4e8ebe0eaf2e8eae520ede0f1e8ebe8ff2e
        
-12.438238,-10.242093,-1.2998546,-5.962235,-13.427248,-12.878796,-16.230131,-17.201805,-1.0009803,-24.018206,-23.622522,-5.2191486,-4.1623316,-11.167844,-13.7162,-26.255186,-2.1109056,0.028141916,-3.6518037,1.7760476,-7.4237423,-13.722433,-3.9797523,-12.691779,21.79608,-13.254122,-0.24368778,-16.654087,-0.72549474,-7.4011917,-5.029045,-7.342651,-3.902788,-2.1215475,-17.628601,4.9885983,-9.793963
 [...]
-IBM500 
c38583894085a2a340a49540a385a7a38540859540c5c2c3c4c9c36b4081a58583408485a2408183838595a3a27a4051404440484b
      
-7.6649594,-5.1355267,-0.059511315,15.283144,6.6421494,6.853642,10.779206,10.726931,14.810395,-10.952078,-24.543068,-11.163412,0.65159315,-14.676382,-17.094334,-29.665552,-10.920232,-19.944944,-10.267533,1.8090932,-8.857017,5.5395036,0.2454361,-10.220937,-3.9662151,-13.020994,-1.419175,-23.684883,-2.9006724,-22.812782,-24.694105,-11.838569,-5.818771,-2.858652,-14.0329075,-2.4
 [...]
-UTF-16LE-emoji 
53006d0069006c0069006e0067002000660061006300650020003dd800de2c0020006e0075006d00620065007200200031002c0030007800300039002c002a000d000a00
        
-15.003139,-17.027376,-3.51826,-5.1978855,-10.307344,-10.29958,-16.53273,-16.69455,-3.9325695,-9.80203,-23.2849,0.0851226,-7.799486,-19.536577,-15.215047,-18.650007,-9.40576,-9.536814,-7.5072374,-3.7153225,-5.7540827,-12.313613,-8.299613,-12.684108,-5.6328754,-11.970228,-2.3121173,-12.124944,-4.9368258,-3.026019,-13.056631,-10.524815,-13.
 [...]
-UTF-16BE-thai  0e2a0e270e310e2a0e140e350e040e230e310e1a        
-11.964084,-20.762892,-3.1328537,-18.356253,-8.522627,-8.178322,-15.8935585,-16.665577,-19.966204,-11.75348,-4.8390594,-7.3139324,-13.296401,-18.660704,-11.799979,-15.680888,-11.938468,-4.528831,-15.936991,-7.375014,-5.5594296,-17.357899,-4.5168223,-5.2730837,-2.0711021,-9.265343,-7.2035103,-11.161968,-15.081558,2.4231293,-3.8285866,-6.273818,-14.419546,-13.408076,-6.7828083,-5.356897,-13.405363,-12.566665,8.712523,3.4173558,23.47274
 [...]
-UTF-32LE       
48000000650000006c0000006c0000006f00000020000000770000006f000000720000006c000000640000002000000000f60100
        
-17.05875,-14.528189,-15.666076,-0.61441404,-7.322275,-7.337266,-11.801676,-11.66604,3.42458,-15.463148,-25.011389,-3.5088234,-0.9584222,-26.449154,-28.674969,-18.125229,-9.030397,-15.152163,-14.510708,1.0818806,-3.7085783,-14.689876,-8.415785,-23.006014,-11.733483,-11.460316,-8.38566,-24.200026,-0.6148938,-12.609276,-14.809252,-22.7466,-22.387167,-9.81124,-7.145131,-5.720913
 [...]
-UTF-16LE-long  
4c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f0020007000610073007300200074006800650020
 [...]
-random 
2dbf9f901511db1b9499ddf6b72b5c3544cadbf085b0a22369a741858d27322dc88a3686eb481d5d30a2528592acb75beeed34672f90906a585185ab9bfaa434a38cded677964e3a9c3cd417e59921a304e491d98507bab2995b7f66121591971321ff775c9e026ca0aa5ee871cfe23fa67357f22aafc210cdb80826827b12d5a21c2a6461ebd0ed58838b9bb8f55aa769ab474b26c427c92a32878780877582901e974cef0ef669b3895fcd82d65b30d5db4fa866d00257fa2532702604a3db50b3c67e9451c85ef5d28c9059c1ea2ee7612b2f3be3480c1a96c02a905a43250647774d82f3e16e353d63fc0c001aac9041a
 [...]
+windows-1252   
4c6520636166e920657374207472e87320626f6e2c206d657263692062656175636f757020706f757220766f747265206163637565696c20e02050617269732e
        
-3.728935,-5.657221,-8.863847,-11.482648,-9.561532,-8.604757,-7.43752,-7.47787,-11.954477,-0.52729106,-3.5210848,-6.8066764,-5.745957,0.95063055,-2.6654189,4.4273067,-3.8865416,-9.069194,-3.4056673,-2.6009703,-6.928908,-13.549944,-3.4414327,-1.8783902,1.049321,7.4346766,-0.76980925,9.129718,-2.3844194,-4.8255363,1.1439289,4.446611,-1.823241,-3.9001
 [...]
+UTF-8  
4772c3bcc39f6520617573204dc3bc6e6368656e20e2809420736368c3b66e65204772c3bcc39f652c20e697a5e69cace8aa9e2e
        
-2.4913938,-13.793381,-1.4572061,-14.358084,-9.161243,-9.23574,-17.909801,-18.012604,-10.497832,-14.088515,-5.1198583,-0.6564211,-1.7893723,-15.199613,-8.216926,-13.226288,-7.448959,-15.409586,-17.54502,-0.383887,-8.473598,-1.7661183,14.962193,-8.954592,-12.808847,-19.34087,-20.458302,-14.473593,-18.670933,-20.235914,-11.93831,-9.55483,-4.0152555,-10.492821,-11.533362,-3.9111333
 [...]
+Shift_JIS      
93fa967b8cea82cc8365834c8358836782c582b7814282b182ea82cd83658358836782c582b78142
        
2.57346,-15.059127,4.1472445,4.398974,3.2613506,2.9596803,-1.2703773,-1.1718445,5.013065,-11.654519,-1.2685089,-10.18289,1.9499466,-18.974812,-4.3752637,-12.470514,-6.9975033,-22.48132,-12.230818,-2.341249,-11.824933,23.784927,-0.3636093,-10.2057905,-23.064056,-22.359735,-13.014805,-28.699781,-6.697913,-16.720522,-17.511112,-25.210554,-15.232973,-20.280565,-5.8799047,3.1145117,-5.4519334,-17.457754
 [...]
+ISO-8859-7     b6f1e8f1e120e3e9e120f4e7ed20f0f1fcebe7f8e720f4e7f220e2dfe1f22e  
-4.2260146,-7.166552,4.272812,-3.6654096,-17.119062,-17.21886,-10.04226,-10.850519,-1.8220028,-11.028378,-14.251019,-1.2311823,-11.025501,-15.914036,-11.57087,-16.756527,-1.6853925,-6.3898015,14.812312,-1.487704,-0.98001575,-9.570157,-18.158705,-17.645805,-6.141889,-8.478981,10.022151,-18.13142,-1.2372838,-2.2352679,-21.749323,-8.896761,-8.972828,3.966666,-5.2044406,1.3768202,-8.522958,-22.16721,-3.6683455,-5.35420
 [...]
+windows-1251   
cdeee2e0ff20eff0eee3f0e0ecece020efee20eff0eef4e8ebe0eaf2e8eae520ede0f1e8ebe8ff2e
        
-4.3219924,-2.6516016,-9.340967,-12.152876,-13.966185,-13.472875,-13.55826,-13.810733,-13.358068,-11.727482,-21.428669,-3.2278848,-5.659675,-1.6961,-7.5128765,-10.297179,-1.32963,-0.46188512,-5.089943,-0.10025105,0.9076651,-25.003895,-7.767557,-5.3756065,22.452518,-8.460794,-2.3472595,-11.188715,-2.718852,-2.2678545,-13.817303,-3.5131617,-3.0150795,0.7813312,-22.011662,3.3440535,-3.1379137,-14.3
 [...]
+IBM500 
c38583894085a2a340a49540a385a7a38540859540c5c2c3c4c9c36b4081a58583408485a2408183838595a3a27a4051404440484b
      
-1.7550082,-4.214464,-0.5797895,16.546648,7.4590564,7.533397,10.255611,10.165785,16.111078,-10.486634,-10.018218,-6.621841,-0.5470758,-19.307821,-19.399403,-12.256375,-10.52631,-13.840994,-9.651697,-3.8074477,-1.8314303,0.8553792,-4.122628,-16.783268,-6.5785995,-11.44013,-6.879944,-20.1175,-8.461582,-1.1915878,-25.546877,-17.379385,-10.510866,-6.510443,-8.374821,0.7822373,-6.
 [...]
+UTF-16LE-emoji 
53006d0069006c0069006e0067002000660061006300650020003dd800de2c0020006e0075006d00620065007200200031002c0030007800300039002c002a000d000a00
        
-15.549664,-24.161572,-9.073898,-17.394,-16.21087,-15.716525,-17.243904,-17.603678,-14.58064,-8.127118,-1.9639423,-3.2922075,-10.522701,-12.247312,-4.403795,-14.619513,-3.590005,-10.321466,-9.084791,-1.8683473,-4.0146136,-13.013371,-10.761813,-4.666911,-3.0703082,-10.834128,-2.3034816,-8.052512,-1.7130973,-13.0478945,-7.2918253,-8.470657,
 [...]
+UTF-16BE-thai  0e2a0e270e310e2a0e140e350e040e230e310e1a        
-8.29257,-16.64819,-4.35965,-6.4487634,-7.919255,-8.070051,-16.993357,-16.577213,-6.2294827,-5.588318,0.15071312,-4.5574737,-8.368986,-11.317027,-3.2231424,-7.0018835,-2.9365144,-18.20875,-9.931014,-1.5361279,-4.716726,-7.872422,-2.8873098,-5.687302,-12.370205,-7.995459,-10.747399,-11.426267,-9.6712055,-13.869811,-10.653317,-12.601955,-7.9654365,-8.82498,-6.027836,-9.065274,-13.249044,-7.864153,6.855218,0.112247616,13.834446,-15.1515
 [...]
+UTF-32LE       
48000000650000006c0000006c0000006f00000020000000770000006f000000720000006c000000640000002000000000f60100
        
-15.230365,-16.029194,-9.992692,-14.304272,-18.323326,-17.793623,-13.888347,-13.490578,-11.84498,-2.6002812,-5.9193563,-5.3789177,-18.504791,-16.726011,-22.108696,-15.171628,-10.669345,-21.244684,-7.5246096,-5.7280064,-12.366646,-25.123335,-17.600935,-19.455034,-8.627289,-3.2324917,-1.897337,-3.4269407,-9.871976,-18.790285,-10.662663,-2.746646,-13.5467205,1.89393,-12.476049,-
 [...]
+UTF-16LE-long  
4c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f0020007000610073007300200074006800650020
 [...]
+random 
2dbf9f901511db1b9499ddf6b72b5c3544cadbf085b0a22369a741858d27322dc88a3686eb481d5d30a2528592acb75beeed34672f90906a585185ab9bfaa434a38cded677964e3a9c3cd417e59921a304e491d98507bab2995b7f66121591971321ff775c9e026ca0aa5ee871cfe23fa67357f22aafc210cdb80826827b12d5a21c2a6461ebd0ed58838b9bb8f55aa769ab474b26c427c92a32878780877582901e974cef0ef669b3895fcd82d65b30d5db4fa866d00257fa2532702604a3db50b3c67e9451c85ef5d28c9059c1ea2ee7612b2f3be3480c1a96c02a905a43250647774d82f3e16e353d63fc0c001aac9041a
 [...]
diff --git 
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkFamilies.java
 
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkFamilies.java
index 749e93e534..efed7ee301 100644
--- 
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkFamilies.java
+++ 
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/JunkFamilies.java
@@ -26,6 +26,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.tika.ml.junkdetect.HtmlContentCleaner;
+
 /** Synthetic junk for training: wrong decodes of clean text, and the 
pointwise junk families. */
 final class JunkFamilies {
 
@@ -63,6 +65,17 @@ final class JunkFamilies {
 
     static final int EXTRA_DECODES = 2;
 
+    /**
+     * UTF-16 misreads per text, both directions: its BOM-less UTF-16 bytes 
read as windows-1252
+     * (what Mojibuster offers against a lone UTF-16 pick: every other char 
NUL or a C0 control) and
+     * its single-byte/UTF-8 bytes read as UTF-16 (ideograph soup).
+     */
+    static final int WIDE_DECODES = 1;
+
+    private static final List<Charset> WIDE = 
List.of(StandardCharsets.UTF_16LE, StandardCharsets.UTF_16BE);
+
+    private static final Charset WINDOWS_1252 = 
Charset.forName("windows-1252");
+
     private JunkFamilies() {
     }
 
@@ -91,9 +104,22 @@ final class JunkFamilies {
         extra.removeAll(decoders);
         extra.remove(source);
         out.addAll(misreads(b, text, extra, r, EXTRA_DECODES));
+        out.addAll(wideMisreads(b, text, r));
         return out;
     }
 
+    /** Cleaned as the runtime cleans a candidate decode (NUL -> U+FFFD), so 
train and inference match. */
+    private static List<String> wideMisreads(byte[] b, String text, Random r) {
+        List<String> out = new ArrayList<>();
+        for (Charset wide : WIDE) {
+            out.add(HtmlContentCleaner.clean(new String(text.getBytes(wide), 
WINDOWS_1252)));
+            out.add(HtmlContentCleaner.clean(new String(b, wide)));
+        }
+        out.removeIf(m -> m == null || m.isEmpty() || m.equals(text));
+        Collections.shuffle(out, r);
+        return out.size() > WIDE_DECODES ? new ArrayList<>(out.subList(0, 
WIDE_DECODES)) : out;
+    }
+
     private static List<String> misreads(byte[] b, String text, 
Iterable<String> decoders, Random r, int max) {
         List<String> out = new ArrayList<>();
         for (String d : decoders) {
diff --git a/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh 
b/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
index c349dcc0a2..75fffa7cd8 100755
--- a/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
+++ b/tika-ml/tika-ml-junkdetect-tools/train-junk-model.sh
@@ -29,7 +29,7 @@
 
 set -euo pipefail
 
-CORPUS=junk-clean-20261103                                   # Wikipedia 
sentences per language
+CORPUS=junk-clean-20261111                                   # Wikipedia 
sentences per language + arv (vocalized Arabic, Wikisource train buckets)
 GOLD=charsets/gold-v113-train-halves.tsv                     # gold v1.13 
cc-html train, even/odd halves, damage spans
 SAMPLE=charsets/cchtml-train-unanimous/sample-gen-20261103.tsv  # cc-html 
train pages for real-decision pairs
 SAMPLE_DOCS=charsets/cchtml-train-unanimous/docs
diff --git 
a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin
 
b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin
index 1a90b56965..0c1824e28d 100644
Binary files 
a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin
 and 
b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin
 differ
diff --git 
a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin.provenance
 
b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin.provenance
index 5f9bd0132c..9f55e72fa9 100644
--- 
a/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin.provenance
+++ 
b/tika-ml/tika-ml-junkdetect/src/main/resources/org/apache/tika/ml/junkdetect/junkdetect.bin.provenance
@@ -1,48 +1,48 @@
-model_sha256   0f4d6535d1cae74b74892adc43298732e399393cde8d0b2235bf729cae7388a6
-trained_at     2026-11-08T01:29:35.867917407Z
-source_corpus  junk-clean-20261103
-source_manifest_sha256 
cad620363ea1a4eceac5775197b62068795531930a759493b644caafe930ca4a
-source_content_sha256  
32c371b35d7b00291e7cf1d1f7c305cf1db56908c76d2f42a1ac80f8d0c530d2
-training_data  20261107-201733-5bc3f3034004/built
-training_data_manifest_sha256  
176b567837c33ca0dd16b6f8879539b0e54193be259923616e946e0b5952a4b7
-git_commit     5bc3f30340046c78c11a1bcb6eca75992a8ca695
-args   --corpus data/junk-clean-20261103 --built 
20261107-201733-5bc3f3034004/built --gold charsets/gold-v113-train-halves.tsv 
--sample cchtml-train-unanimous/sample-gen-20261103.tsv --sample-docs 
cchtml-train-unanimous/docs --output junkdetect/junkdetect.bin --git-commit 
5bc3f30340046c78c11a1bcb6eca75992a8ca695
+model_sha256   c410b0bd5cfa1b2dd1047528b67207748a601a674fa3fb408ddc77685f0be4e9
+trained_at     2026-11-12T18:40:57.982796303Z
+source_corpus  junk-clean-20261111
+source_manifest_sha256 
51fe915afdd9ea27a6d153d1a04560b90a63e1f22ed27b73b825872f42579f05
+source_content_sha256  
bb870f6c20919178320e3a7995858ae187fe6caf3839dbe7a806671886375916
+training_data  20261112-132853-3ed1521aa902/built
+training_data_manifest_sha256  
3ebd0e5ce051e333f8ca5f9d55c5eafbb37828a6520d871326f0a122034dc154
+git_commit     3ed1521aa902877af20b8e6fbdb161e199f85b78
+args   --corpus data/junk-clean-20261111 --built 
20261112-132853-3ed1521aa902/built --gold charsets/gold-v113-train-halves.tsv 
--sample cchtml-train-unanimous/sample-gen-20261103.tsv --sample-docs 
cchtml-train-unanimous/docs --output junkdetect/junkdetect.bin --git-commit 
3ed1521aa902877af20b8e6fbdb161e199f85b78
 java_version   25.0.4.1
 format JUNKHNG1
 gold   charsets/gold-v113-train-halves.tsv
 gold_sha256    88c28a193f2b405ff668cec233e65dca9d1755b1e18af62d3fb4385246fe08c8
 real_decision_sample   cchtml-train-unanimous/sample-gen-20261103.tsv
 real_decision_sample_sha256    
6366919ca3ab598046b18cb3e6d1dc4b050ee14614c80ccbc47d4fa59a7ef1e1
-source_corpus_dir      data/junk-clean-20261103
-wiki_units     96667
-wiki_units_upper_case_added    6144
+source_corpus_dir      data/junk-clean-20261111
+wiki_units     97714
+wiki_units_upper_case_added    6147
 wiki_units_without_language    0.0000
-pairs_wiki     548682
+pairs_wiki     658417
 gold_pages_even        13293
 gold_pages_odd 13239
 chain  
BOMDetector,MetadataCharsetDetector,MojibusterEncodingDetector,HtmlEncodingDetector
 real_decision_pages    66022
 real_decision_variants 280708
 real_decision_errors   0
-pairs_real_decision_fit        7540
-pairs_real_decision_select     973
+pairs_real_decision_fit        6505
+pairs_real_decision_select     796
 pairs_gold_decoders_odd        254125
-pairs_gold_chain_odd   825
+pairs_gold_chain_odd   818
 defer_sweep_pages      13293
-defer_sweep_decisive_pages     96
-defer_sweep_curve      0.25:+1/-0 0.50:+5/-0 0.75:+8/-0 1.00:+11/-1 
1.25:+12/-2 1.50:+15/-2 1.75:+16/-2 2.00:+20/-2 2.25:+21/-5 2.50:+22/-5 
2.75:+22/-7 3.00:+25/-7 3.25:+26/-8 3.50:+26/-9 3.75:+29/-10 4.00:+29/-14 
4.25:+30/-14 4.50:+32/-15 4.75:+33/-15 5.00:+33/-21 5.25:+34/-22 5.50:+34/-22 
5.75:+34/-25 6.00:+34/-28 6.25:+34/-35 6.50:+34/-38 6.75:+34/-41 7.00:+34/-45 
7.25:+34/-46 7.50:+34/-48 7.75:+34/-49 8.00:+34/-51 8.25:+35/-55 8.50:+35/-55 
8.75:+35/-57 9.00:+35/-57 9.25:+35/-58 9.50:+35/- [...]
-defer_sweep_at_threshold       +20/-2
+defer_sweep_decisive_pages     103
+defer_sweep_curve      0.25:+3/-0 0.50:+5/-0 0.75:+7/-1 1.00:+11/-1 
1.25:+20/-2 1.50:+23/-2 1.75:+24/-2 2.00:+28/-3 2.25:+29/-3 2.50:+29/-6 
2.75:+30/-6 3.00:+33/-6 3.25:+34/-8 3.50:+35/-8 3.75:+35/-11 4.00:+35/-13 
4.25:+38/-14 4.50:+39/-14 4.75:+41/-17 5.00:+41/-21 5.25:+41/-21 5.50:+41/-22 
5.75:+41/-23 6.00:+41/-27 6.25:+41/-32 6.50:+41/-37 6.75:+41/-38 7.00:+41/-41 
7.25:+41/-45 7.50:+41/-45 7.75:+41/-49 8.00:+41/-53 8.25:+41/-53 8.50:+41/-53 
8.75:+41/-57 9.00:+41/-59 9.25:+41/-60 9.50:+41/- [...]
+defer_sweep_at_threshold       +28/-3
 pairs_gold_decoders_even       254294
-pairs_gold_chain_even  761
-point_clean_wiki       102857
+pairs_gold_chain_even  795
+point_clean_wiki       103837
 point_clean_web        12692
-point_damaged_windows  1698
-point_damaged_clean_rest       196
-point_damaged_stale_pages      1
-point_examples 246362
+point_damaged_windows  1644
+point_damaged_clean_rest       170
+point_damaged_stale_pages      27
+point_examples 249944
 trained_scripts        [LATIN, GREEK, CYRILLIC, ARMENIAN, HEBREW, ARABIC, 
DEVANAGARI, BENGALI, GURMUKHI, GUJARATI, ORIYA, TAMIL, TELUGU, KANNADA, 
MALAYALAM, SINHALA, THAI, LAO, TIBETAN, MYANMAR, GEORGIAN, HANGUL, ETHIOPIC, 
KHMER, HIRAGANA, KATAKANA, HAN, NKO, OL_CHIKI]
 deferral_threshold     2.0000
-train_seconds  651
+train_seconds  653
 config.HashedTrainingConfig.LOG2_SIZE  21
 config.HashedTrainingConfig.MAX_N      3
 config.HashedTrainingConfig.CLIP_CHARS 6000
diff --git 
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HashedArbitrationTest.java
 
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HashedArbitrationTest.java
index b856b420da..02e2479772 100644
--- 
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HashedArbitrationTest.java
+++ 
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HashedArbitrationTest.java
@@ -84,7 +84,9 @@ public class HashedArbitrationTest {
 
     /**
      * From a CC page declared iso-8859-3 but written in Latin-1 with UTF-8 
fragments: 0xC3 is undefined
-     * in ISO-8859-3, so its decode has U+FFFD the windows-1252 decode lacks.
+     * in ISO-8859-3, so its decode has U+FFFD the windows-1252 decode lacks. 
The rule is checked with the
+     * pinned test model; which decode the bundled model prefers is a thin 
margin (|logit| < 0.3 across
+     * retrains, deferral threshold 2) and belongs to the gold chain gate, not 
here.
      */
     @Test
     public void declarationWithReplacementCharactersIsNotAnAnchor() throws 
Exception {
@@ -100,11 +102,11 @@ public class HashedArbitrationTest {
         texts.put(iso3, new String(page, iso3));
         assertTrue(texts.get(iso3).indexOf('�') >= 0);
 
-        HashedJunkModel bundled = JunkDetector.loadFromClasspath().getModel();
-        HashedArbitration.Decision d = HashedArbitration.decide(bundled, 
texts, null, iso3);
-        assertEquals(W1252, d.getChampion());
-        assertNull(d.getAnchor());
-        assertEquals(W1252, d.pick(Double.MAX_VALUE));
+        HashedArbitration.Decision d = HashedArbitration.decide(model, texts, 
null, iso3);
+        assertEquals(champion(texts), d.getChampion());
+        // the declared decode is an anchor only when it is the champion itself
+        assertEquals(d.getChampion().equals(iso3) ? iso3 : null, 
d.getAnchor());
+        assertEquals(d.getChampion(), d.pick(Double.MAX_VALUE));
 
         // a champion with as many replacement characters leaves the 
declaration as the anchor
         Map<Charset, String> same = new LinkedHashMap<>();
@@ -124,9 +126,20 @@ public class HashedArbitrationTest {
         fewer.put(W1252, new String(stray, W1252));
         fewer.put(iso3, new String(stray, iso3));
         assertTrue(fewer.get(W1252).indexOf('�') >= 0);
-        HashedArbitration.Decision f = HashedArbitration.decide(bundled, 
fewer, null, iso3);
-        assertEquals(W1252, f.getChampion());
-        assertNull(f.getAnchor());
+        HashedArbitration.Decision f = HashedArbitration.decide(model, fewer, 
null, iso3);
+        assertEquals(champion(fewer), f.getChampion());
+        assertEquals(f.getChampion().equals(iso3) ? iso3 : null, 
f.getAnchor());
+    }
+
+    /** The candidate with the highest pair value, first on ties. */
+    private Charset champion(Map<Charset, String> texts) {
+        Charset best = null;
+        for (Map.Entry<Charset, String> e : texts.entrySet()) {
+            if (best == null || model.pairValue(e.getValue()) > 
model.pairValue(texts.get(best))) {
+                best = e.getKey();
+            }
+        }
+        return best;
     }
 
     @Test
diff --git 
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/LoneWideArbitrationTest.java
 
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/LoneWideArbitrationTest.java
new file mode 100644
index 0000000000..e4ef6d6550
--- /dev/null
+++ 
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/LoneWideArbitrationTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import org.apache.tika.detect.EncodingDetectorContext;
+import org.apache.tika.detect.EncodingResult;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.ml.chardetect.MojibusterEncodingDetector;
+import org.apache.tika.parser.ParseContext;
+
+/**
+ * A statistical UTF-16/32 pick is never Mojibuster's only candidate: 
windows-1252 is offered as the
+ * alternative decode and JunkFilter keeps whichever reads as text.  The 
assertions hold whatever the
+ * model's own pick is, so a retrain cannot break them by chance.
+ */
+public class LoneWideArbitrationTest {
+
+    private static final Charset WIN1252 = Charset.forName("windows-1252");
+    private static MojibusterEncodingDetector mojibuster;
+    private static JunkFilterEncodingDetector junkFilter;
+
+    @BeforeAll
+    static void load() throws Exception {
+        mojibuster = new MojibusterEncodingDetector();
+        junkFilter = new 
JunkFilterEncodingDetector(JunkDetector.loadFromClasspath().getModel());
+    }
+
+    /** Mojibuster then JunkFilter, as DefaultEncodingDetector runs them. */
+    private static Charset chain(byte[] bytes) throws Exception {
+        EncodingDetectorContext ctx = new EncodingDetectorContext();
+        ParseContext pc = new ParseContext();
+        pc.set(EncodingDetectorContext.class, ctx);
+        try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+            List<EncodingResult> moji = mojibuster.detect(tis, new Metadata(), 
pc);
+            assertFalse(moji.isEmpty());
+            if (moji.get(0).getResultType() == 
EncodingResult.ResultType.STATISTICAL
+                    && isWide(moji.get(0).getCharset())) {
+                assertTrue(moji.stream().anyMatch(r -> 
!isWide(r.getCharset())),
+                        "statistical wide pick returned alone: " + moji);
+            }
+            ctx.addResult(moji, "MojibusterEncodingDetector");
+            List<EncodingResult> out = junkFilter.detect(tis, new Metadata(), 
pc);
+            return (out.isEmpty() ? moji : out).get(0).getCharset();
+        }
+    }
+
+    private static boolean isWide(Charset cs) {
+        return cs.name().startsWith("UTF-16") || 
cs.name().startsWith("UTF-32");
+    }
+
+    private static String ircLog() {
+        StringBuilder sb = new StringBuilder();
+        String[] nicks = {"<alice>", "<bob>", "<carol>"};
+        for (int i = 0; i < 40; i++) {
+            sb.append(nicks[i % 3]).append(" the build on \u0003").append(4 + 
i % 9).append("branch-")
+              .append(i).append("\u0003 is \u0002green\u000f again, see ticket 
").append(1000 + i)
+              .append(" for the \u001funderlined\u000f notes\r\n");
+        }
+        return sb.toString();
+    }
+
+    private static String dosBanner() {
+        StringBuilder sb = new StringBuilder("\u0007\u0007 WELCOME TO THE 
BOARD \u0007\u0007\r\n");
+        for (int i = 0; i < 30; i++) {
+            sb.append("\u0001 item ").append(i).append(": press any key to 
continue the download \u0014\r\n");
+        }
+        sb.append("\u001a");
+        return sb.toString();
+    }
+
+    @Test
+    public void asciiWithControlsIsNotUtf16() throws Exception {
+        for (String text : new String[] {ircLog(), dosBanner()}) {
+            byte[] bytes = text.getBytes(StandardCharsets.US_ASCII);
+            Charset pick = chain(bytes);
+            assertFalse(isWide(pick), pick.name());
+            // 7-bit bytes decode identically under every ASCII superset; the 
label does not matter
+            assertEquals(new String(bytes, WIN1252), new String(bytes, pick));
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"UTF-16LE", "UTF-16BE"})
+    public void bomlessUtf16TextStaysUtf16(String charset) throws Exception {
+        String[] texts = {
+            "The committee met on Tuesday to review the budget for the coming 
year. Members agreed that "
+            + "the library extension should proceed, and that the café on the 
ground floor would reopen.",
+            "北京是中华人民共和国的首都,也是全国的政治和文化中心。这座城市有三千多年的建城历史。",
+            "وقال الوزير إن الحكومة ستعمل على تحسين الخدمات الصحية في جميع 
المناطق خلال العام المقبل."
+        };
+        Charset cs = Charset.forName(charset);
+        for (String text : texts) {
+            assertEquals(cs, chain(text.getBytes(cs)), text.substring(0, 12));
+        }
+    }
+}
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectionTest.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectionTest.java
index 1af0d0f8ce..94d3db2e3c 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectionTest.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/html/HtmlEncodingDetectionTest.java
@@ -126,6 +126,67 @@ public class HtmlEncodingDetectionTest extends TikaTest {
         assertContains("有什么需要我帮你的", r.xml);
     }
 
+    /**
+     * UTF-8 BOM, ASCII bytes, and a meta that says UTF-16BE (shape of a 
Common Crawl page).  The
+     * visible text is sparse ASCII art, so the junk margin alone is too thin 
to overrule the meta.
+     */
+    @Test
+    public void testUtf16MetaOnAsciiBytesIsUtf8() throws Exception {
+        String html = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+                + "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 
Transitional//EN\"\n"
+                + "    
\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\";>\n"
+                + "<html xmlns=\"http://www.w3.org/1999/xhtml\"; lang=\"en\" 
xml:lang=\"en\">\n"
+                + "<head>\n"
+                + "    <title>Example Studio</title>\n"
+                + "    <meta http-equiv=\"Content-Type\" content=\"text/html; 
charset=utf-16be\" />\n"
+                + "    <link rel=\"stylesheet\" type=\"text/css\" 
href=\"style.css\" />\n"
+                + "    <meta name=\"keywords\" content=\"example, studio, 
design, media, art, gallery,"
+                + " video, photography\" />\n"
+                + "</head>\n"
+                + "<body>\n"
+                + "    ##<div id=\"page\"> # # ## ## ## ## ## ## # # # # # # # 
#\n"
+                + "    #<br />\n"
+                + "    <div id=\"stage\"><br />\n"
+                + "    Back soon.\n"
+                + "    " + "#<br />".repeat(11) + "\n"
+                + "    </div>#\n"
+                + "    <div id=\"menu\"></div>#\n"
+                + "    </div>#\n"
+                + " #<br />\n"
+                + "</body>\n"
+                + "</html>\n\n";
+        byte[] body = html.getBytes(StandardCharsets.US_ASCII);
+        // even length: an odd trailing byte would give the UTF-16 decode a 
U+FFFD and hide the bug
+        assertEquals(0, body.length % 2);
+        byte[] bytes = new byte[body.length + 3];
+        bytes[0] = (byte) 0xEF;
+        bytes[1] = (byte) 0xBB;
+        bytes[2] = (byte) 0xBF;
+        System.arraycopy(body, 0, bytes, 3, body.length);
+        Metadata metadata = new Metadata();
+        XMLResult r;
+        try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+            r = getXML(tis, AUTO_DETECT_PARSER, metadata);
+        }
+        assertEquals("UTF-8", r.metadata.get(HttpHeaders.CONTENT_ENCODING));
+        assertContains("Back soon.", r.xml);
+    }
+
+    /** A real UTF-16 page cannot carry an ASCII-readable meta, so its BOM 
still decides. */
+    @Test
+    public void testUtf16BomWithUtf16MetaStaysUtf16() throws Exception {
+        String html = "\uFEFF<html><head><meta 
charset=\"utf-16\"><title>t</title></head>"
+                + "<body><p>Ce texte est en UTF-16 avec une marque 
d\u2019ordre des octets.</p>"
+                + "</body></html>";
+        Metadata metadata = new Metadata();
+        XMLResult r;
+        try (TikaInputStream tis = 
TikaInputStream.get(html.getBytes(StandardCharsets.UTF_16LE))) {
+            r = getXML(tis, AUTO_DETECT_PARSER, metadata);
+        }
+        assertEquals("UTF-16LE", r.metadata.get(HttpHeaders.CONTENT_ENCODING));
+        assertContains("marque d\u2019ordre des octets", r.xml);
+    }
+
     /** TIKA-1980. */
     @Test
     public void testAllHeadElements() throws Exception {

Reply via email to