This is an automated email from the ASF dual-hosted git repository.

tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new fee8015695 TIKA-5062: classify magic-less content as text or binary 
with TextBinaryClassifier (#3598)
fee8015695 is described below

commit fee8015695e795424a17383098695db3bd74fe67
Author: Tim Allison <[email protected]>
AuthorDate: Thu Nov 12 14:02:33 2026 -0500

    TIKA-5062: classify magic-less content as text or binary with 
TextBinaryClassifier (#3598)
---
 CHANGES.txt                                        |  10 +
 .../apache/tika/detect/TextBinaryClassifier.java   | 597 +++++++++++++++++++++
 .../main/java/org/apache/tika/mime/MimeTypes.java  |  10 +-
 .../tika/detect/TextBinaryClassifierTest.java      | 159 ++++++
 .../org/apache/tika/mime/MimeDetectionTest.java    |  49 +-
 5 files changed, 819 insertions(+), 6 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index fa3dd54880..4db7145361 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -130,6 +130,16 @@ Release 4.3.0 - ???
      A UTF-8 BOM followed by binary magic (JPEG, PNG, GIF, ID3, zip, SQLite) 
is detected by
      that magic instead of as text (TIKA-5029).
 
+   * Content with no magic is judged text or binary by TextBinaryClassifier, a 
small
+     model over byte statistics of the 64 KB prefix, instead of TextDetector's 
control-byte
+     check, which only accepted mostly-ASCII or valid UTF-8. Extensionless 
text in legacy
+     encodings (windows-1251, GB18030, Shift_JIS, EUC-JP, EUC-KR, KOI8-R and 
others), logs
+     with a few control bytes, zero-filled holes or backspace progress bars 
are now
+     text/plain; 3.x returned application/octet-stream for these, and 4.0-4.2 
rescued
+     some only through a 512-byte re-sniff. Printable ASCII is always text; 
inputs under
+     16 bytes keep the old check. A text name or Content-Type still loses to 
content with
+     5% or more control bytes that the classifier calls binary (TIKA-5062).
+
    * New or fixed magic: Emacs bytecode from every Emacs version (only version 
19 matched
      before, and *.elc is no longer also an octet-stream glob), Lua bytecode
      (application/x-lua-bytecode), Qt .qm translations 
(application/x-qt-translation, were
diff --git 
a/tika-core/src/main/java/org/apache/tika/detect/TextBinaryClassifier.java 
b/tika-core/src/main/java/org/apache/tika/detect/TextBinaryClassifier.java
new file mode 100644
index 0000000000..bbd75bb31b
--- /dev/null
+++ b/tika-core/src/main/java/org/apache/tika/detect/TextBinaryClassifier.java
@@ -0,0 +1,597 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+/**
+ * Decides whether magic-less bytes are text or binary. A binned logistic 
model (a scorecard)
+ * over byte statistics of the prefix: control-byte classes, NUL structure, 
printable runs, line
+ * shape, byte-class periodicity, entropy, UTF-8 validity and UTF-16 
plausibility. It needs no
+ * charset detector, so legacy 8-bit and CJK text count as text, and so do 
text with a few
+ * controls, zero-filled holes and backspace progress bars.
+ * <p>
+ * Trained outside this repository on typed text in many encodings (with 
control, NUL-hole and
+ * separator augmentations) against mid-file windows of typed binaries, 
magic-less binaries and
+ * synthesized fills, samples, image rows, tables and object code. The arrays 
are generated; do
+ * not edit them by hand. Inputs shorter than {@link #MIN_LENGTH} bytes carry 
too little evidence
+ * and are not classified.
+ *
+ * @since Apache Tika 4.3.0
+ */
+public final class TextBinaryClassifier {
+
+    /** Shortest input the model decides. */
+    public static final int MIN_LENGTH = 16;
+
+    // generated by the training scripts (35 features, 8 quantile bins); do 
not edit by hand
+    static final String[] FEATURES = {
+            "logn", "nul", "bs", "esc", "c0", "del", "c1", "ff", "hi", 
"ctlNoNul", "nulRuns", "nulRunMax",
+            "nulRunMean", "nulMod2", "nulMod4", "nulAfterStr", "nulStr", 
"cov8a", "nlKB", "lineMean", "lineStd",
+            "lineMax", "bsCtx", "ac1", "ac2", "ac4", "ac8", "ac16", "ent", 
"distinct", "topB", "u8ok",
+            "u8badKB", "u16", "u16asc"
+    };
+    private static final double[][] EDGES = {
+            { // logn
+                    5.754887502163469, 7.149747119504682, 8.396604781181859, 
9.505811553919594, 10.689997971419444,
+                    11.870846298570346, 13.406005627992485
+            },
+            { // nul
+                    0.019995632340095118, 0.39709629708827826
+            },
+            { // bs
+                    0.004129666834030079
+            },
+            { // esc
+                    0.002523128679562658
+            },
+            { // c0
+                    0.0568959761244983, 0.1076923076923077
+            },
+            { // del
+                    0.001491465316532892
+            },
+            { // c1
+                    0.001404371180760891, 0.03362508201802201, 
0.122322099538035
+            },
+            { // ff
+                    0.00473533621330323
+            },
+            { // hi
+                    0.0038095238095238095, 0.046875, 0.22629537237525812, 
0.4921052631578947, 0.6565917638464245
+            },
+            { // ctlNoNul
+                    0.006213592366837315, 0.07654376183463077, 
0.12507791780548536
+            },
+            { // nulRuns
+                    8.258064516129032, 87.31782945736434
+            },
+            { // nulRunMax
+                    1.0, 6.0
+            },
+            { // nulRunMean
+                    1.0, 2.2860107190286474
+            },
+            { // nulMod2
+                    0.5116279069767442, 0.6199380165289257
+            },
+            { // nulMod4
+                    0.2769366379930956, 0.4
+            },
+            { // nulAfterStr
+                    0.3915307971014493
+            },
+            { // nulStr
+                    0.28871331948096
+            },
+            { // cov8a
+                    0.06691937157305528, 0.3911804509845034, 
0.8625976971706535, 0.9880588755860057
+            },
+            { // nlKB
+                    4.308630945998772, 11.407705673074272, 19.729053904812396, 
28.779619808866116, 50.25766871165644
+            },
+            { // lineMean
+                    14.766729152465349, 24.37291666666667, 34.875, 46.0, 65.0, 
106.66641025641026, 203.703125
+            },
+            { // lineStd
+                    4.499728862888747, 14.648999369161627, 24.874367443102166, 
38.066371371830726, 62.818474464887444,
+                    124.01573926622027
+            },
+            { // lineMax
+                    34.0, 63.0, 89.0, 138.0, 236.0, 437.0, 842.0
+            },
+            { // bsCtx
+                    0.5
+            },
+            { // ac1
+                    -0.014645433732129898, -6.090915087741955e-06, 0.0, 
0.001156642184634793, 0.02923237875234655,
+                    0.1125418152391145, 0.23822469498055948
+            },
+            { // ac2
+                    -0.0023151378088784383, 0.0, 0.005819579583405954, 
0.05187415943023149, 0.14083653862136364,
+                    0.272730637787365
+            },
+            { // ac4
+                    -0.003707768722685391, 0.0, 0.002868279756737202, 
0.03738516597302008, 0.1123482449642961,
+                    0.2386099113685777
+            },
+            { // ac8
+                    -0.0033944490193483015, 0.0, 0.0028463679233658723, 
0.031422249339583846, 0.09236417472085226,
+                    0.21638970257424897
+            },
+            { // ac16
+                    -0.0071732051701677915, 0.0, 0.0009436492081413839, 
0.017246907046564786, 0.06897695306082857,
+                    0.18520800220809217
+            },
+            { // ent
+                    3.2225571723615913, 4.033423537031595, 4.560147688419784, 
4.886989850387823, 5.1522139434457435,
+                    5.4761576479361, 6.398792231420389
+            },
+            { // distinct
+                    19.0, 33.0, 49.0, 64.0, 80.0, 108.0, 180.0
+            },
+            { // topB
+                    0.034482758620689655, 0.05834046056850623, 
0.07209393453379019, 0.08769149498151083,
+                    0.11184713998273697, 0.17624892670160258, 
0.49778684328405964
+            },
+            { // u8ok
+                    0.10157727153361344, 0.19414620217018846, 
0.45454545454545453
+            },
+            { // u8badKB
+                    0.4183080190778277, 38.12977237838997, 187.60011920317265, 
423.73148812770967, 517.8892901075354
+            },
+            { // u16
+                    0.5000750610134621, 0.7641788885433941, 
0.8219416554456218, 0.9028357862511449, 0.9428571428571428,
+                    0.967741935483871, 0.9984012636289396
+            },
+            { // u16asc
+                    0.002168507640888958, 0.037037037037037035
+            },
+    };
+    private static final double[][] WEIGHTS = {
+            { // logn
+                    -0.2310552191574832, -0.21454777147190596, 
-0.18720285013968138, 0.04043538182810273,
+                    -0.0011347842473640145, 0.5098808748046423, 
0.029937496947774873, -0.19424181002493615
+            },
+            { // nul
+                    -0.5534189838509629, -0.08852605572828375, 
0.3940163581184254
+            },
+            { // bs
+                    0.941705619863977, -1.1896343013248327
+            },
+            { // esc
+                    0.36700426674966113, -0.6149329482105139
+            },
+            { // c0
+                    1.287796145711941, -0.5192731668499568, -1.0164516603229834
+            },
+            { // del
+                    0.586816300178283, -0.8347449816391818
+            },
+            { // c1
+                    1.3015976755135448, 0.2136216550141253, 
-0.5161059143125614, -1.2470420976759873
+            },
+            { // ff
+                    0.12174317826350628, -0.3696718597243321
+            },
+            { // hi
+                    0.14965444085693166, 0.39476484585952565, 
0.49127485791474845, -0.6483281868735459,
+                    -0.6504227957585362, 0.015128156540113825
+            },
+            { // ctlNoNul
+                    1.4289802643302496, -0.517994133979678, 
-0.7021363665739889, -0.4567784452375745
+            },
+            { // nulRuns
+                    -0.2564679234529926, -0.18592516182962976, 
0.19446440382178015
+            },
+            { // nulRunMax
+                    1.9328632906237004, -0.31834881031328466, 
-1.8624431617713795
+            },
+            { // nulRunMean
+                    1.9328632906237004, -0.5923858075934905, 
-1.5884061644911696
+            },
+            { // nulMod2
+                    -0.099713586614645, 0.0553038928090881, -0.2035189876553327
+            },
+            { // nulMod4
+                    -0.24256003738837195, -0.48982048848977344, 
0.4844518444172544
+            },
+            { // nulAfterStr
+                    -0.48829944089571653, 0.2403707594349252
+            },
+            { // nulStr
+                    -0.9037103991283869, 0.6557817176675811
+            },
+            { // cov8a
+                    -0.6880486969786129, -0.49266761809079457, 
-0.922031077877108, 0.06470512914568709,
+                    1.790113582340125
+            },
+            { // nlKB
+                    -0.9060205391104922, -0.9380839356393983, 
-0.16574032676983722, 0.15709931992082168,
+                    0.6350006412382854, 0.9698161588998079
+            },
+            { // lineMean
+                    0.5535694600631904, 0.14193797040403583, 
0.38109679748301095, 0.48135015034424317,
+                    0.36036675494371245, -0.03584748431925305, 
-0.8152337160441433, -1.3151686143356447
+            },
+            { // lineStd
+                    -0.7022035733239248, -0.017669433007049717, 
0.17268461707830748, 0.4208108855370347,
+                    -0.0344312513639432, 0.2593099080938668, 
-0.3464298344751901
+            },
+            { // lineMax
+                    -0.4337125077993913, -0.08815454486161163, 
0.06141618379144683, -0.04202722613795733,
+                    0.07121522082134446, 0.26525175243975657, 
0.12090224080376981, -0.20281980051821652
+            },
+            { // bsCtx
+                    -0.4745241498412075, 0.22659546838031475
+            },
+            { // ac1
+                    0.9940394538443768, 0.5541923503449178, 
0.08048984474590326, -0.646209691683989,
+                    0.28041195331372165, -0.600813029811643, 
-0.4254032648676291, -0.48463629734650665
+            },
+            { // ac2
+                    -0.028937001938560303, -0.2902408813910141, 
-0.41246666619131167, -0.2128073200748067,
+                    0.06428475002166702, 0.6435252301020274, 
-0.01128679198883952
+            },
+            { // ac4
+                    -0.1963096243653022, -0.722004648858803, 
-0.5702104194327913, -0.3201496153710012,
+                    0.06811452345543635, 0.2946020742824756, 1.1980290288291586
+            },
+            { // ac8
+                    -0.2737512069109041, -0.7329743062802071, 
-0.4364693017213012, 0.061542494945294283,
+                    0.0698718810231144, 0.3678820055016415, 0.6959697519815295
+            },
+            { // ac16
+                    -0.12311810995138345, -0.3447769275861189, 
-0.15218494178719055, 0.30226365606927824,
+                    0.032025834015999774, 0.03991020286289722, 
-0.002048395084322618
+            },
+            { // ent
+                    -0.31972906512940474, 0.6303748680407597, 
0.21202965361914433, -0.1607538910083972,
+                    0.21406369819694765, 0.2975981081786166, 
-0.18325798844721863, -0.9382540649112939
+            },
+            { // distinct
+                    -2.1659173174584496, -1.14341367167224, 
-0.38619106903632294, -0.1777969091663309,
+                    0.5302827815999743, 0.5894067720107392, 
1.2356538328103999, 1.2700468994513778
+            },
+            { // topB
+                    -1.5281763000720574, -0.5288595602473192, 
0.1300143991973351, 0.7603002812320029, 0.53037160171491,
+                    0.8424515460638258, 0.6464507608068175, -1.1004814101563625
+            },
+            { // u8ok
+                    -0.9270157212910567, -0.5388113512829429, 
0.6593092993660308, 0.558589091747253
+            },
+            { // u8badKB
+                    0.6191097559302005, 0.5257182368960559, 
0.5988965161121214, -0.35933937610436184,
+                    -0.7089828107082297, -0.9233310035865361
+            },
+            { // u16
+                    0.6021631366019026, 0.053120195059327815, 
-0.41057615520909246, -0.6069578121702731,
+                    -0.23291750610992767, 0.15458659610644337, 
0.6607799310113494, -0.46812706675057586
+            },
+            { // u16asc
+                    0.041099389670909856, -0.06469694129777277, 
-0.22433112983399436
+            },
+    };
+    private static final double INTERCEPT = -0.3937321772678982;
+    static final double THRESHOLD = 0.8377535594988522;
+
+    private static final int NFEAT = FEATURES.length;
+
+    private TextBinaryClassifier() {
+    }
+
+    /**
+     * @return true if the bytes are printable ASCII or the model reads them 
as text; false for
+     *         binary or input under {@link #MIN_LENGTH}
+     */
+    public static boolean isText(byte[] data) {
+        if (data.length < MIN_LENGTH) {
+            return false;
+        }
+        // printable ASCII only: text by definition (the trainer dropped such 
binary windows)
+        return isPrintableAscii(data) || probability(data) >= THRESHOLD;
+    }
+
+    private static boolean isPrintableAscii(byte[] data) {
+        for (byte b : data) {
+            int c = b & 0xFF;
+            if (c >= 0x7F || (c < 0x20 && c != 9 && c != 10 && c != 12 && c != 
13)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * @return estimated probability that the bytes are text (calibrated on 
held-out data)
+     */
+    public static double probability(byte[] data) {
+        if (data.length == 0) {
+            return 0;
+        }
+        return score(features(data));
+    }
+
+    static double score(double[] f) {
+        double z = INTERCEPT;
+        for (int j = 0; j < NFEAT; j++) {
+            z += WEIGHTS[j][bin(EDGES[j], f[j])];
+        }
+        return 1.0 / (1.0 + Math.exp(-z));
+    }
+
+    // number of interior edges <= x, as the trainer bins
+    private static int bin(double[] interior, double x) {
+        int k = 0;
+        while (k < interior.length && interior[k] <= x) {
+            k++;
+        }
+        return k;
+    }
+
+    private static boolean texty(int c) {
+        return c == 9 || c == 10 || c == 13 || (c >= 0x20 && c < 0x7F) || c >= 
0x80;
+    }
+
+    private static boolean asciiPrintable(int c) {
+        return c == 9 || c == 10 || c == 13 || (c >= 0x20 && c < 0x7F);
+    }
+
+    // 0 NUL, 1 control, 2 other ASCII, 3 high byte
+    private static final int[] CLASS = new int[256];
+
+    static {
+        for (int c = 0; c < 256; c++) {
+            CLASS[c] = c == 0 ? 0
+                    : (c < 0x20 && c != 9 && c != 10 && c != 13 && c != 12) || 
c == 0x7F ? 1
+                    : c < 0x80 ? 2 : 3;
+        }
+    }
+
+    private static boolean u16Text(int u) {
+        return u == 9 || u == 10 || u == 13 || (u >= 0x20 && u < 0x7F) || (u 
>= 0xA0 && u < 0x2C00)
+                || (u >= 0x3000 && u < 0xA000) || (u >= 0xAC00 && u < 0xD7A4) 
|| (u >= 0xFF00 && u < 0xFFF0);
+    }
+
+    private static final int[] LAGS = {1, 2, 4, 8, 16};
+
+    /** Feature vector in {@link #FEATURES} order; definitions must match the 
trainer exactly. */
+    static double[] features(byte[] b) {
+        int n = b.length;
+        double nd = n;
+        int[] h = new int[256];
+        int[] nulPos4 = new int[4];
+        int[] cls = new int[4];
+        int[] agree = new int[LAGS.length];
+        int[] u16ok = new int[2];
+        int[] u16asc = new int[2];
+        int nulRuns = 0;
+        int nulRunMax = 0;
+        long nulRunTot = 0;
+        int nulAfterStr = 0;
+        int nulStr = 0;
+        int run = 0;
+        int tRun = 0;
+        int aRun = 0;
+        int lastTextyRun = 0;
+        long cov8a = 0;
+        int bsCtx = 0;
+        int nl = 0;
+        int lineMax = 0;
+        int cur = 0;
+        double lsum = 0;
+        double lsq = 0;
+        int lcount = 0;
+        for (int i = 0; i < n; i++) {
+            int c = b[i] & 0xFF;
+            h[c]++;
+            int k = CLASS[c];
+            cls[k]++;
+            for (int l = 0; l < LAGS.length; l++) {
+                if (i >= LAGS[l] && CLASS[b[i - LAGS[l]] & 0xFF] == k) {
+                    agree[l]++;
+                }
+            }
+            if (c == 0) {
+                nulPos4[i & 3]++;
+                if (run == 0) {
+                    nulRuns++;
+                    if (tRun >= 4) {
+                        nulAfterStr++;
+                    }
+                    lastTextyRun = tRun;
+                }
+                run++;
+            } else if (run > 0) {
+                nulRunTot += run;
+                nulRunMax = Math.max(nulRunMax, run);
+                if (lastTextyRun >= 4 && texty(c)) {
+                    nulStr++;
+                }
+                run = 0;
+            }
+            if (c != 0 && texty(c)) {
+                tRun++;
+            } else {
+                tRun = 0;
+            }
+            if (asciiPrintable(c)) {
+                aRun++;
+            } else {
+                if (aRun >= 8) {
+                    cov8a += aRun;
+                }
+                aRun = 0;
+            }
+            if (c == 8) {
+                int pv = i > 0 ? b[i - 1] & 0xFF : 0;
+                int nx = i + 1 < n ? b[i + 1] & 0xFF : 0;
+                if ((pv == 8 || (pv != 0 && texty(pv))) && (nx == 8 || (nx != 
0 && texty(nx)))) {
+                    bsCtx++;
+                }
+            }
+            if (c == '\n') {
+                nl++;
+            }
+            if (c == '\n' || (c == '\r' && (i + 1 >= n || b[i + 1] != '\n'))) {
+                lsum += cur;
+                lsq += (double) cur * cur;
+                lcount++;
+                lineMax = Math.max(lineMax, cur);
+                cur = 0;
+            } else if (c != '\r') {
+                cur++;
+            }
+            if ((i & 1) == 1) {
+                int lo = b[i - 1] & 0xFF;
+                int le = lo | c << 8;
+                int be = lo << 8 | c;
+                if (u16Text(le)) {
+                    u16ok[0]++;
+                }
+                if (u16Text(be)) {
+                    u16ok[1]++;
+                }
+                if (c == 0 && asciiPrintable(lo)) {
+                    u16asc[0]++;
+                }
+                if (lo == 0 && asciiPrintable(c)) {
+                    u16asc[1]++;
+                }
+            }
+        }
+        if (run > 0) {
+            nulRunTot += run;
+            nulRunMax = Math.max(nulRunMax, run);
+        }
+        if (aRun >= 8) {
+            cov8a += aRun;
+        }
+        lineMax = Math.max(lineMax, cur);
+        lsum += cur;
+        lsq += (double) cur * cur;
+        lcount++;
+
+        int u8ok = 0;
+        int u8bad = 0;
+        for (int i = 0; i < n; ) {
+            int c = b[i] & 0xFF;
+            if (c < 0x80) {
+                i++;
+                continue;
+            }
+            int k = c >= 0xF0 && c <= 0xF4 ? 3 : c >= 0xE0 ? 2 : c >= 0xC2 && 
c < 0xE0 ? 1 : -1;
+            if (c > 0xF4) {
+                k = -1;
+            }
+            if (k < 0) {
+                u8bad++;
+                i++;
+                continue;
+            }
+            if (i + k >= n) {
+                // sequence cut by the end of the prefix
+                u8ok += n - i;
+                break;
+            }
+            boolean ok = true;
+            for (int j = 1; j <= k; j++) {
+                if ((b[i + j] & 0xC0) != 0x80) {
+                    ok = false;
+                    break;
+                }
+            }
+            if (ok) {
+                u8ok += k + 1;
+                i += k + 1;
+            } else {
+                u8bad++;
+                i++;
+            }
+        }
+
+        int nul = h[0];
+        int hi = cls[3];
+        int c0 = 0;
+        for (int c = 1; c < 32; c++) {
+            if (c != 9 && c != 10 && c != 13 && c != 12 && c != 8 && c != 27) {
+                c0 += h[c];
+            }
+        }
+        int c1 = 0;
+        for (int c = 0x80; c < 0xA0; c++) {
+            c1 += h[c];
+        }
+        double ent = 0;
+        int distinct = 0;
+        int top = 0;
+        for (int c = 0; c < 256; c++) {
+            if (h[c] > 0) {
+                distinct++;
+                double p = h[c] / nd;
+                ent -= p * Math.log(p) / Math.log(2);
+                if (c != 32 && c != 10 && c != 13 && c != 9) {
+                    top = Math.max(top, h[c]);
+                }
+            }
+        }
+        double exp = 0;
+        for (int k = 0; k < 4; k++) {
+            exp += Math.pow(cls[k] / nd, 2);
+        }
+        double lm = lsum / lcount;
+        int units = n / 2;
+        double[] ac = new double[LAGS.length];
+        for (int l = 0; l < LAGS.length; l++) {
+            int m = Math.max(0, n - LAGS[l]);
+            ac[l] = m == 0 ? 0.0 : agree[l] / (double) m - exp;
+        }
+        int nm2 = Math.max(nulPos4[0] + nulPos4[2], nulPos4[1] + nulPos4[3]);
+        int nm4 = Math.max(Math.max(nulPos4[0], nulPos4[1]), 
Math.max(nulPos4[2], nulPos4[3]));
+        return new double[] {
+                Math.log(n) / Math.log(2),
+                nul / nd,
+                h[8] / nd,
+                h[27] / nd,
+                c0 / nd,
+                h[0x7F] / nd,
+                c1 / nd,
+                h[0xFF] / nd,
+                hi / nd,
+                (h[8] + h[27] + c0 + h[0x7F]) / nd,
+                nulRuns * 1024.0 / nd,
+                nulRunMax,
+                nulRuns == 0 ? 0.0 : (double) nulRunTot / nulRuns,
+                nul == 0 ? 0.5 : (double) nm2 / nul,
+                nul == 0 ? 0.25 : (double) nm4 / nul,
+                nulRuns == 0 ? 0.0 : (double) nulAfterStr / nulRuns,
+                nulRuns == 0 ? 0.0 : (double) nulStr / nulRuns,
+                cov8a / nd,
+                nl * 1024.0 / nd,
+                lm,
+                Math.sqrt(Math.max(0, lsq / lcount - lm * lm)),
+                lineMax,
+                h[8] == 0 ? 0.0 : (double) bsCtx / h[8],
+                ac[0], ac[1], ac[2], ac[3], ac[4],
+                ent,
+                distinct,
+                top / nd,
+                hi == 0 ? 1.0 : u8ok / (double) (u8ok + u8bad),
+                u8bad * 1024.0 / nd,
+                units == 0 ? 0 : Math.max(u16ok[0], u16ok[1]) / (double) units,
+                units == 0 ? 0 : Math.max(u16asc[0], u16asc[1]) / (double) 
units,
+        };
+    }
+}
diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java 
b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
index 70910abd66..c4a7f48a1c 100644
--- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
+++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java
@@ -37,6 +37,7 @@ import javax.xml.namespace.QName;
 import org.apache.tika.Tika;
 import org.apache.tika.annotation.TikaComponent;
 import org.apache.tika.detect.Detector;
+import org.apache.tika.detect.TextBinaryClassifier;
 import org.apache.tika.detect.TextDetector;
 import org.apache.tika.detect.TextStatistics;
 import org.apache.tika.detect.XmlRootExtractor;
@@ -306,10 +307,15 @@ public final class MimeTypes implements Detector, 
Serializable {
             return result;
         }
 
-        // Finally, assume plain text if no control bytes are found
+        byte[] content = withoutNulTail(data);
+        if (content.length >= TextBinaryClassifier.MIN_LENGTH) {
+            return TextBinaryClassifier.isText(content) ? 
Collections.singletonList(textMimeType)
+                    : rootMimeTypeL;
+        }
+        // too short for the classifier: plain text only if no control bytes 
are found
         try {
             TextDetector detector = new TextDetector(getMinLength());
-            try (TikaInputStream tis = 
TikaInputStream.get(withoutNulTail(data))) {
+            try (TikaInputStream tis = TikaInputStream.get(content)) {
                 MimeType type = forName(detector.detect(tis, new Metadata(), 
new ParseContext()).toString());
                 return Collections.singletonList(type);
             }
diff --git 
a/tika-core/src/test/java/org/apache/tika/detect/TextBinaryClassifierTest.java 
b/tika-core/src/test/java/org/apache/tika/detect/TextBinaryClassifierTest.java
new file mode 100644
index 0000000000..f8acc6a8dd
--- /dev/null
+++ 
b/tika-core/src/test/java/org/apache/tika/detect/TextBinaryClassifierTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.detect;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Random;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+import org.apache.tika.mime.MimeTypes;
+import org.apache.tika.parser.ParseContext;
+
+public class TextBinaryClassifierTest {
+
+    private static final String RUSSIAN = "Банк России сообщает, что с 1 июля 
изменяются правила расчёта "
+            + "обязательных резервов для кредитных организаций.\r\nПросим 
довести настоящее письмо до "
+            + "сведения всех заинтересованных лиц и обеспечить его исполнение 
в установленный срок.\r\n";
+    private static final String CHINESE = "关于做好二〇一九年度预算执行情况审计工作的通知\r\n各区县人民政府,"
+            + "市政府各部门:为进一步加强财政资金管理,提高资金使用效益,现将有关事项通知如下。\r\n"
+            + "一、高度重视审计工作,切实落实主体责任。二、严格按照规定时限报送材料。\r\n";
+    private static final String JAPANESE = 
"第三十回全国大会のご案内\n日時:二〇一八年十月十三日(土)午前十時より\n"
+            + "会場:東京国際フォーラム ホールB\n参加をご希望の方は、事務局までお申し込みください。\n"
+            + "詳しくは当会ホームページをご覧ください。皆様のご参加をお待ちしております。\n";
+    private static final String ENGLISH = "Release notes for version 2.4.1\n\n"
+            + "Fixed a crash when the configuration file contains an empty 
section.\n"
+            + "The installer now keeps existing settings when upgrading in 
place.\n"
+            + "Thanks to everyone who reported problems on the mailing 
list.\n";
+
+    private static byte[] repeat(byte[] unit, int times) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        for (int i = 0; i < times; i++) {
+            out.writeBytes(unit);
+        }
+        return out.toByteArray();
+    }
+
+    private static void assertText(byte[] data) {
+        assertTrue(TextBinaryClassifier.isText(data), "p=" + 
TextBinaryClassifier.probability(data));
+    }
+
+    private static void assertBinary(byte[] data) {
+        assertFalse(TextBinaryClassifier.isText(data), "p=" + 
TextBinaryClassifier.probability(data));
+    }
+
+    @Test
+    public void legacyAndUnicodeText() {
+        assertText(repeat(RUSSIAN.getBytes(Charset.forName("windows-1251")), 
4));
+        assertText(repeat(CHINESE.getBytes(Charset.forName("GB18030")), 4));
+        assertText(repeat(JAPANESE.getBytes(Charset.forName("EUC-JP")), 4));
+        assertText(repeat(JAPANESE.getBytes(Charset.forName("Shift_JIS")), 4));
+        assertText(repeat((RUSSIAN + CHINESE + ENGLISH).getBytes(UTF_8), 3));
+    }
+
+    @Test
+    public void textWithControls() {
+        // a stray bell and form feed in an otherwise plain log
+        String log = "2019-03-04 10:15:02 INFO  starting worker pool size=8\n"
+                + "2019-03-04 10:15:03 WARN  disk usage at 91% on /var\u0007\n"
+                + "2019-03-04 10:15:09 INFO  checkpoint written in 412 ms\n\f"
+                + "2019-03-04 10:16:00 INFO  shutting down\n";
+        assertText(repeat(log.getBytes(US_ASCII), 6));
+
+        // a preallocated log whose middle block was never written
+        byte[] hole = repeat(log.getBytes(US_ASCII), 12);
+        Arrays.fill(hole, hole.length / 3, hole.length / 3 + 600, (byte) 0);
+        assertText(hole);
+
+        // an installer log with a backspace-driven progress counter
+        StringBuilder sb = new StringBuilder("Extracting files to C:\\Program 
Files\\Example\r\n");
+        for (int pct = 0; pct <= 100; pct += 5) {
+            sb.append(String.format(java.util.Locale.ROOT, "%3d%%", 
pct)).append("\b\b\b\b");
+        }
+        sb.append("\r\nInstallation finished. Restart the computer to complete 
setup.\r\n");
+        assertText(repeat(sb.toString().getBytes(US_ASCII), 4));
+    }
+
+    @Test
+    public void binary() {
+        byte[] random = new byte[8192];
+        new Random(5029).nextBytes(random);
+        assertBinary(random);
+
+        byte[] fill = repeat(new byte[] {(byte) 0xC7, (byte) 0xC7, (byte) 
0xC7, (byte) 0xC7, (byte) 0xC7,
+                (byte) 0xC7, (byte) 0xC7, (byte) 0xC7, (byte) 0xC7, (byte) 
0xC7, '\n'}, 40);
+        assertBinary(fill);
+
+        byte[] pcm = new byte[4000];
+        for (int i = 0; i < pcm.length; i++) {
+            pcm[i] = (byte) (128 + 90 * Math.sin(2 * Math.PI * 440 * i / 
8000.0));
+        }
+        assertBinary(pcm);
+
+        // Elf64_Sym records: name, info, other, shndx, value, size
+        ByteBuffer sym = ByteBuffer.allocate(24 * 
64).order(ByteOrder.LITTLE_ENDIAN);
+        for (int i = 0; i < 64; i++) {
+            sym.putInt(1 + 17 * i).put((byte) 0x12).put((byte) 
0).putShort((short) 14)
+                    .putLong(0x401000L + 0x40L * i).putLong(32 + i);
+        }
+        assertBinary(sym.array());
+    }
+
+    @Test
+    public void printableAsciiIsText() {
+        // a low-diversity payload the model alone scores binary (p 0.79)
+        byte[] runs = "XXXXXXXXXXYYYYYYYYYY".getBytes(US_ASCII);
+        assertTrue(TextBinaryClassifier.probability(runs) < 0.837);
+        assertText(runs);
+    }
+
+    @Test
+    public void shortInputIsNotClassified() {
+        assertFalse(TextBinaryClassifier.isText("fifteen 
bytes!!".getBytes(US_ASCII)));
+    }
+
+    @Test
+    public void mimeTypesUsesClassifier() throws Exception {
+        MimeTypes types = MimeTypes.getDefaultMimeTypes();
+        byte[] russian = 
repeat(RUSSIAN.getBytes(Charset.forName("windows-1251")), 4);
+        assertEquals(MediaType.TEXT_PLAIN, detect(types, russian));
+        byte[] fill = repeat(new byte[] {(byte) 0xC7, (byte) 0xC7, (byte) 
0xC7, (byte) 0xC7, '\n'}, 60);
+        assertEquals(MediaType.OCTET_STREAM, detect(types, fill));
+        // 15 bytes stay on the control-byte check
+        assertEquals(MediaType.TEXT_PLAIN, detect(types, "fifteen 
bytes!!".getBytes(US_ASCII)));
+        assertEquals(MediaType.OCTET_STREAM, detect(types, new byte[] {1, 2, 
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}));
+    }
+
+    private static MediaType detect(MimeTypes types, byte[] data) throws 
Exception {
+        try (TikaInputStream tis = TikaInputStream.get(data)) {
+            return types.detect(tis, new Metadata(), new ParseContext());
+        }
+    }
+}
diff --git 
a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java 
b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
index 0800216d86..c98a7c076c 100644
--- a/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
+++ b/tika-core/src/test/java/org/apache/tika/mime/MimeDetectionTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.tika.mime;
 
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_16BE;
 import static java.nio.charset.StandardCharsets.UTF_16LE;
@@ -27,6 +28,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.Random;
 
 import org.apache.commons.io.ByteOrderMark;
 import org.junit.jupiter.api.BeforeAll;
@@ -451,10 +457,6 @@ public class MimeDetectionTest {
     public void testBinaryContentRefusesTextGlob() throws IOException {
         // headers copied from real files that a text glob used to claim
         String[][] cases = {
-                // Terminal Reality POD archive
-                {"Bulldog.pod", 
"2f000000436f6d70696c65642062792042696e456469742056657273696f6e20"
-                        + 
"322e322e38207da62d2900000000000000000000000000000000000000000000"
-                        + 
"00000000000000000000000000000000000000004152545c4241545453352e41"},
                 // Mozilla startupCache XDR bytecode, named for its source
                 {"AccountManager.js",
                         
"0000001c000000b9000000c5fcc6bf73b9000000000000000000000000000000"
@@ -482,6 +484,45 @@ public class MimeDetectionTest {
         }
     }
 
+    @Test
+    public void testTerminalRealityPodIsBinary() throws IOException {
+        byte[] pod = terminalRealityPod();
+        assertEquals(MediaType.OCTET_STREAM, detect(pod, "Bulldog.pod"));
+        Metadata metadata = new Metadata();
+        metadata.set(HttpHeaders.CONTENT_TYPE, "text/plain");
+        try (TikaInputStream tis = TikaInputStream.get(pod)) {
+            assertEquals(MediaType.OCTET_STREAM, MIME_TYPES.detect(tis, 
metadata, new ParseContext()));
+        }
+    }
+
+    // shaped like real BinEdit .pod archives: entry count, comment, 40-byte 
directory entries, indexed pixels
+    private static byte[] terminalRealityPod() {
+        ByteBuffer bb = 
ByteBuffer.allocate(16384).order(ByteOrder.LITTLE_ENDIAN);
+        int entries = 48;
+        bb.putInt(entries);
+        bb.put("Compiled by BinEdit Version 2.2.8 }".getBytes(ISO_8859_1));
+        bb.put(new byte[] {(byte) 0xA6, '-', ')'});
+        bb.position(0x54);
+        Random random = new Random(5029);
+        int offset = 0x54 + 40 * entries;
+        for (int i = 0; i < entries; i++) {
+            String name = String.format(Locale.ROOT, "ART\\%02dBF%02d.%s", 
random.nextInt(100), i / 2,
+                    i % 2 == 0 ? "ACT" : "RAW");
+            byte[] padded = Arrays.copyOf(name.getBytes(US_ASCII), 32);
+            int size = i % 2 == 0 ? 0x300 : 0x10000;
+            bb.put(padded).putInt(size).putInt(offset);
+            offset += size;
+        }
+        while (bb.hasRemaining()) {
+            byte value = (byte) random.nextInt(256);
+            int run = 1 + random.nextInt(24);
+            for (int k = 0; k < run && bb.hasRemaining(); k++) {
+                bb.put(value);
+            }
+        }
+        return bb.array();
+    }
+
     @Test
     public void testTextGlobStillWinsForText() throws IOException {
         String latin = "The quick brown fox jumps over the lazy dog.\n";

Reply via email to