This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new b739f6f257 TIKA-5027: junk detector treats scripts without a usable
table as unmodeled, not junk (#3522)
b739f6f257 is described below
commit b739f6f25700f2a0736a737c02cb2672ec60c94b
Author: Tim Allison <[email protected]>
AuthorDate: Wed Nov 4 15:16:04 2026 -0500
TIKA-5027: junk detector treats scripts without a usable table as
unmodeled, not junk (#3522)
Scripts whose bigram table has fewer than 1000 bigrams add no z1 evidence;
all-unmodeled text takes the coherence fallback. TextQualityScore gains
getCoverage(); tika-eval reports languageness as unknown at coverage 0.
The fallback's z6 term is anchored so letter-free text scores -1, not +3.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 7 ++
.../org/apache/tika/quality/TextQualityScore.java | 29 +++++-
.../org/apache/tika/eval/app/ProfilerBase.java | 2 +-
.../eval/core/metadata/TikaEvalMetadataFilter.java | 2 +-
.../tika/ml/junkdetect/tools/TrainJunkModel.java | 7 +-
.../ml/junkdetect/JunkDetectorRoundTripTest.java | 6 +-
.../apache/tika/ml/junkdetect/BigramTables.java | 3 +-
.../apache/tika/ml/junkdetect/JunkDetector.java | 89 +++++++++++++----
.../tika/ml/junkdetect/UnmodeledScriptTest.java | 109 +++++++++++++++++++++
9 files changed, 221 insertions(+), 33 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 5167d60514..1294b4607e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -102,6 +102,13 @@ Release 4.3.0 - ???
OTHER CHANGES
+ * The junk detector no longer scores text in scripts its model has no
usable table
+ for (Yi, Mongolian, Canadian syllabics, Cherokee, Adlam and 37 others) as
+ junk: it scored -3 to -19 before. TextQualityScore gains getCoverage(),
and
+ tika-eval reports languageness for such text as unknown. Text with no
letters
+ (whitespace, digits, symbols) now scores -1 instead of +3, below clean
prose
+ (TIKA-5027).
+
* The file-to-markdown-docker agent skill turns OCR off with an empty
text-recognizers
list and a server block (its tesseract-ocr-parser config failed to load
since 4.2.0),
and checks tk:parsed-by-full-set, not pdf:ocr-page-count, to confirm OCR
ran. A test
diff --git
a/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java
b/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java
index 8eea89ddec..6dfbf55a50 100644
--- a/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java
+++ b/tika-core/src/main/java/org/apache/tika/quality/TextQualityScore.java
@@ -29,6 +29,10 @@ package org.apache.tika.quality;
* {@code zScore}. For short strings these bounds are wide; for long strings
* they narrow. Prefer {@code ciLow < threshold} over {@code zScore <
threshold}
* when triggering actions, to reduce false positives on short strings.
+ *
+ * <p>{@code coverage} is the fraction of the scored text the detector actually
+ * has a model for. At 0 the score says nothing about language; a consumer
+ * reporting text quality should treat it as unknown.
*/
public final class TextQualityScore {
@@ -40,15 +44,23 @@ public final class TextQualityScore {
private final float ciLow;
private final float ciHigh;
private final String dominantScript;
+ private final float coverage;
public TextQualityScore(float zScore, float pClean,
float ciLow, float ciHigh,
String dominantScript) {
+ this(zScore, pClean, ciLow, ciHigh, dominantScript, Float.NaN);
+ }
+
+ public TextQualityScore(float zScore, float pClean,
+ float ciLow, float ciHigh,
+ String dominantScript, float coverage) {
this.zScore = zScore;
this.pClean = pClean;
this.ciLow = ciLow;
this.ciHigh = ciHigh;
this.dominantScript = dominantScript;
+ this.coverage = coverage;
}
/** Z-score relative to clean text for the detected script. 0 = average
clean; negative = worse. */
@@ -76,6 +88,19 @@ public final class TextQualityScore {
return dominantScript;
}
+ /**
+ * Fraction in [0,1] of the text's scoreable units covered by the
detector's
+ * model; NaN if the text had none (e.g. digits only) or the detector does
not report it.
+ */
+ public float getCoverage() {
+ return coverage;
+ }
+
+ /** True if the text had scoreable units but none in a script the detector
models. */
+ public boolean isUnmodeled() {
+ return coverage == 0f;
+ }
+
/** True if scoring could not be performed (e.g. empty or
unsupported-script input). */
public boolean isUnknown() {
return Float.isNaN(zScore);
@@ -87,7 +112,7 @@ public final class TextQualityScore {
return "TextQualityScore[UNKNOWN script=" + dominantScript + "]";
}
return String.format(java.util.Locale.ROOT,
- "TextQualityScore[z=%.3f p=%.3f ci=(%.3f,%.3f) script=%s]",
- zScore, pClean, ciLow, ciHigh, dominantScript);
+ "TextQualityScore[z=%.3f p=%.3f ci=(%.3f,%.3f) script=%s
coverage=%.3f]",
+ zScore, pClean, ciLow, ciHigh, dominantScript, coverage);
}
}
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
index 748eabcc39..20a03becea 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java
@@ -494,7 +494,7 @@ public abstract class ProfilerBase {
JunkDetector jd = TikaEvalMetadataFilter.getJunkDetector();
if (jd != null && !content.isEmpty()) {
TextQualityScore q = jd.score(content);
- if (!q.isUnknown()) {
+ if (!q.isUnknown() && !q.isUnmodeled()) {
results.put(LanguagenessMarker.class, q.getZScore());
}
}
diff --git
a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java
b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java
index 83b1bd0978..57e38fdfda 100644
---
a/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java
+++
b/tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/metadata/TikaEvalMetadataFilter.java
@@ -129,7 +129,7 @@ public class TikaEvalMetadataFilter extends
MetadataFilterBase {
// Replaces the GLM-based languageness removed in TIKA-4720.
if (JUNK_DETECTOR != null) {
TextQualityScore q = JUNK_DETECTOR.score(content);
- metadata.set(LANGUAGENESS, q.isUnknown() ? -99.0f : q.getZScore());
+ metadata.set(LANGUAGENESS, q.isUnknown() || q.isUnmodeled() ?
-99.0f : q.getZScore());
} else {
metadata.set(LANGUAGENESS, -99.0f);
}
diff --git
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
index 33629e97f2..dc7e5f053e 100644
---
a/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
+++
b/tika-ml/tika-ml-junkdetect-tools/src/main/java/org/apache/tika/ml/junkdetect/tools/TrainJunkModel.java
@@ -277,10 +277,9 @@ public class TrainJunkModel {
// file via forEachScriptBigram (the SAME tokenizer inference uses):
the
// text is split into maximal same-script LETTER runs (marks attach),
each
// run wrapped TOKEN_START..TOKEN_END, and the within-run bigrams
(incl. the
- // ^/$ edges) tallied to the run's script. Non-letters are boundaries
and
- // emit nothing, so the trained tables hold exactly the bigrams
inference
- // scores. The COMMON bucket is now effectively vestigial (only the
rare
- // COMMON-scripted letter lands there).
+ // ^/$ edges) tallied to the run's script. In-run
punctuation/symbols/digits
+ // attach as glue; whitespace and anomalies end a run. The trained
tables
+ // therefore hold exactly the bigrams inference scores.
//
-----------------------------------------------------------------------
System.out.println("\n--- Phase 1: bucket-by-script F1 tables ---");
Map<String, HashMap<Long, long[]>> pairsByScript = new HashMap<>();
diff --git
a/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java
b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java
index 7433820efe..b2080b4b34 100644
---
a/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java
+++
b/tika-ml/tika-ml-junkdetect-tools/src/test/java/org/apache/tika/ml/junkdetect/JunkDetectorRoundTripTest.java
@@ -94,7 +94,7 @@ public class JunkDetectorRoundTripTest {
saveMinimalModel(tables, modelFile);
// Verify the file roundtrips through the loader.
- JunkDetector detector = JunkDetector.loadFromPath(modelFile);
+ JunkDetector detector = JunkDetector.loadFromPath(modelFile, 0);
assertEquals(JunkDetector.VERSION, detector.getModelVersion(), "Loaded
model should match current VERSION");
TextQualityScore score = detector.score("ABAB");
@@ -131,7 +131,7 @@ public class JunkDetectorRoundTripTest {
Path modelFile = tmp.resolve("junk-test-allseen.bin");
saveMinimalModel(tables, modelFile);
- JunkDetector detector = JunkDetector.loadFromPath(modelFile);
+ JunkDetector detector = JunkDetector.loadFromPath(modelFile, 0);
TextQualityScore score = detector.score("ABAB");
assertEquals(expectedRunZ(tables, "ABAB", -5.0f, 1.0f),
score.getZScore(), 0.05f,
@@ -221,7 +221,7 @@ public class JunkDetectorRoundTripTest {
combiner, scriptBuckets, scriptTransTable, scriptTransCal,
modelPath);
// --- 6. Load via JunkDetector and score ---
- JunkDetector detector = JunkDetector.loadFromPath(modelPath);
+ JunkDetector detector = JunkDetector.loadFromPath(modelPath, 0);
assertEquals(JunkDetector.VERSION, detector.getModelVersion(),
"Loaded model should match current VERSION");
assertTrue(detector.knownScripts().contains("LATIN"),
diff --git
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
index 05974bd927..d26592dfee 100644
---
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
+++
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/BigramTables.java
@@ -220,7 +220,8 @@ public final class BigramTables {
unigramQuantMin, unigramQuantMax);
}
- private int occupiedSlots() {
+ /** Number of distinct bigrams stored. */
+ public int occupiedSlots() {
int n = 0;
for (int k : bigramKeys) {
if (k != EMPTY_KEY) n++;
diff --git
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
index b63f8787e8..e6a4a8fc46 100644
---
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
+++
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/JunkDetector.java
@@ -26,6 +26,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
@@ -102,6 +103,13 @@ public final class JunkDetector implements
TextQualityDetector {
// incompatible and rejected at load.
public static final int VERSION = 15;
+ /**
+ * A script table with fewer distinct bigrams than this is too thin to
score:
+ * real text in it would hit the unseen-pair floor on nearly every bigram
and
+ * read as junk. Such scripts are treated as unmodeled (no z1 evidence).
+ */
+ static final int MIN_MODELED_BIGRAMS = 1000;
+
// Feature 1 — per-script open-addressed codepoint-bigram tables.
// No global Bloom: empty-slot is the membership oracle.
private final Map<String, BigramTables> f1TablesByScript;
@@ -110,6 +118,9 @@ public final class JunkDetector implements
TextQualityDetector {
* z1 is the ONLY per-script feature. */
private final Map<String, float[]> calibrations; // script → float[2] {mu,
sigma}
+ /** Scripts whose z1 table is populated enough to score; see {@link
#MIN_MODELED_BIGRAMS}. */
+ private final Set<String> modeledScripts;
+
// Feature 2 — single GLOBAL block-transition table.
// int16-quantized; dequantize via {@code min + (s/65535) * (max - min)}.
private final short[] blockTable;
@@ -156,8 +167,19 @@ public final class JunkDetector implements
TextQualityDetector {
Map<String, BigramTables> f1TablesByScript,
float[] z5Calibration,
float[] z6Calibration,
- float[] z9Calibration) {
+ float[] z9Calibration,
+ int minModeledBigrams) {
this.calibrations = Collections.unmodifiableMap(calibrations);
+ Set<String> modeled = new HashSet<>();
+ for (Map.Entry<String, BigramTables> e : f1TablesByScript.entrySet()) {
+ // COMMON is the glue bucket for COMMON-scripted letters; always
scored.
+ if (calibrations.containsKey(e.getKey())
+ && (COMMON_SCRIPT.equals(e.getKey())
+ || e.getValue().occupiedSlots() >= minModeledBigrams))
{
+ modeled.add(e.getKey());
+ }
+ }
+ this.modeledScripts = Collections.unmodifiableSet(modeled);
this.blockTable = blockTable;
this.blockTableQuant = blockTableQuant;
this.blockCalibration = blockCalibration;
@@ -217,8 +239,13 @@ public final class JunkDetector implements
TextQualityDetector {
* Loads a model from the given file path. The file may be gzipped or raw.
*/
public static JunkDetector loadFromPath(Path path) throws IOException {
+ return loadFromPath(path, MIN_MODELED_BIGRAMS);
+ }
+
+ /** As {@link #loadFromPath(Path)}, with the modeled-script threshold
overridden (toy test models). */
+ static JunkDetector loadFromPath(Path path, int minModeledBigrams) throws
IOException {
try (InputStream is = Files.newInputStream(path)) {
- return load(is);
+ return load(is, minModeledBigrams);
}
}
@@ -273,6 +300,11 @@ public final class JunkDetector implements
TextQualityDetector {
* </pre>
*/
public static JunkDetector load(InputStream rawIs) throws IOException {
+ return load(rawIs, MIN_MODELED_BIGRAMS);
+ }
+
+ /** As {@link #load(InputStream)}, with the modeled-script threshold
overridden (toy test models). */
+ static JunkDetector load(InputStream rawIs, int minModeledBigrams) throws
IOException {
byte[] peek = rawIs.readNBytes(2);
InputStream rest = new java.io.SequenceInputStream(
new java.io.ByteArrayInputStream(peek), rawIs);
@@ -369,7 +401,7 @@ public final class JunkDetector implements
TextQualityDetector {
scriptTransitionTable, scriptTransitionTableQuant,
scriptTransitionCalibration,
scriptBucketIndex, numScriptBuckets, f1TablesByScript,
- z5Calibration, z6Calibration, z9Calibration);
+ z5Calibration, z6Calibration, z9Calibration,
minModeledBigrams);
}
}
@@ -413,12 +445,11 @@ public final class JunkDetector implements
TextQualityDetector {
/**
* {@inheritDoc}
*
- * <p>The text is split into contiguous runs of the same Unicode script.
- * Each run is scored against its own script model. Logits are combined
- * as a byte-count-weighted average, so mixed-script text (e.g. half
- * LATIN, half HAN) is scored fairly without arbitrarily picking one
script.
- * COMMON, INHERITED, and UNKNOWN codepoints (spaces, punctuation, digits)
- * are attached to the preceding script run.
+ * <p>Letter runs are scored against their script's bigram table and the
+ * per-script z1's are combined count-weighted. Punctuation, symbols and
+ * digits inside a run are scored with it; whitespace ends a run. Scripts
+ * outside {@link #knownScripts()} contribute no z1 evidence and lower
+ * {@link TextQualityScore#getCoverage()}.
*/
@Override
public TextQualityScore score(String text) {
@@ -457,9 +488,9 @@ public final class JunkDetector implements
TextQualityDetector {
return new TextQualityComparison(winner, delta, scoreA, scoreB,
labelA, labelB);
}
- /** Returns the set of script names this model knows about. */
+ /** Returns the scripts this model scores; text in any other script
carries no z1 evidence. */
public Set<String> knownScripts() {
- return calibrations.keySet();
+ return modeledScripts;
}
/** Returns the file-format version of the loaded model
@@ -476,7 +507,9 @@ public final class JunkDetector implements
TextQualityDetector {
private static final class Agg {
float z1, z2, z3, z4, z5, z6, z7, z8, z9, logit;
String dominantScript; // null => no scoreable script (NONE
fallback)
+ String dominantUnmodeledScript;
int totalBigrams;
+ float coverage = Float.NaN;
float[] dominantCal1;
}
@@ -486,7 +519,7 @@ public final class JunkDetector implements
TextQualityDetector {
* each script's mean log-prob is calibrated to {@code
z1_S=(mean-mu)/sigma},
* and the per-script z1's are aggregated count-weighted into a single doc
* {@code z1}. z2..z9 are document-level globals. A single global
combiner
- * produces the logit. No runs, no sentinels, no per-script z2/z3.
+ * produces the logit.
*/
private Agg aggregate(String text) {
// NFKC-normalize so inference matches the trainer's tally AND legacy
@@ -508,14 +541,13 @@ public final class JunkDetector implements
TextQualityDetector {
String[] lastScript = {null};
int[] lastB = {Integer.MIN_VALUE};
int[] lastBIdx = {-1};
+ Map<String, int[]> unmodeled = new HashMap<>();
forEachScriptBigram(cps, (script, a, b) -> {
- if (!calibrations.containsKey(script)) {
+ if (!modeledScripts.contains(script)) {
+ unmodeled.computeIfAbsent(script, k -> new int[1])[0]++;
return;
}
BigramTables t = f1TablesByScript.get(script);
- if (t == null) {
- return;
- }
int idxA = (a == lastB[0] && script.equals(lastScript[0]))
? lastBIdx[0] : codepointToIndex(t, a);
int idxB = codepointToIndex(t, b);
@@ -566,6 +598,18 @@ public final class JunkDetector implements
TextQualityDetector {
agg.z8 = Double.isNaN(rawFrag) ? 1f : 1f - (float) rawFrag;
agg.z9 = computeZ9AlternationRatio(text);
agg.totalBigrams = (int) z1Count;
+ long unmodeledCount = 0;
+ int maxUnmodeled = 0;
+ for (Map.Entry<String, int[]> e : unmodeled.entrySet()) {
+ unmodeledCount += e.getValue()[0];
+ if (e.getValue()[0] > maxUnmodeled) {
+ maxUnmodeled = e.getValue()[0];
+ agg.dominantUnmodeledScript = e.getKey();
+ }
+ }
+ if (z1Count + unmodeledCount > 0) {
+ agg.coverage = (float) z1Count / (z1Count + unmodeledCount);
+ }
if (z1Count == 0 || agg.dominantScript == null) {
// No scoreable LETTER at all (zero runs) — doc-level fallback:
@@ -573,10 +617,11 @@ public final class JunkDetector implements
TextQualityDetector {
// (unmodeled coherent script); density=1 coherence=0 -> very
negative.
// z6 is included so a letter-free but FFFD/anomaly-heavy doc
(which can
// no longer flood z1) is still penalized here — no path ignores
the
- // replacement ratio. For unmodeled-but-clean script z6~0, so
it's inert.
+ // replacement ratio. z6 is 1 for anomaly-free text, so (z6 - 1)
keeps
+ // it inert there: all-whitespace -1, coherent unmodeled script
~+2.4.
agg.dominantScript = null;
agg.z1 = Float.NaN;
- agg.logit = -7f + 4f * agg.z7 + 6f * agg.z8 + 4f * agg.z6;
+ agg.logit = -7f + 4f * agg.z7 + 6f * agg.z8 + 4f * (agg.z6 - 1f);
return agg;
}
agg.z1 = (float) (weightedZ1 / z1Count);
@@ -589,12 +634,13 @@ public final class JunkDetector implements
TextQualityDetector {
Agg a = aggregate(text);
float pClean = (float) (1.0 / (1.0 + Math.exp(-a.logit)));
if (a.dominantScript == null) {
- return new TextQualityScore(a.logit, pClean, a.logit, a.logit,
"NONE");
+ String script = a.dominantUnmodeledScript != null ?
a.dominantUnmodeledScript : "NONE";
+ return new TextQualityScore(a.logit, pClean, a.logit, a.logit,
script, a.coverage);
}
float uncertainty = (a.dominantCal1 != null && a.totalBigrams > 0)
? (float) (1.96 * a.dominantCal1[1] /
Math.sqrt(a.totalBigrams)) : 0f;
return new TextQualityScore(a.logit, pClean,
- a.logit - uncertainty, a.logit + uncertainty,
a.dominantScript);
+ a.logit - uncertainty, a.logit + uncertainty,
a.dominantScript, a.coverage);
}
/**
@@ -609,8 +655,9 @@ public final class JunkDetector implements
TextQualityDetector {
}
Agg a = aggregate(text);
if (a.dominantScript == null) {
+ String script = a.dominantUnmodeledScript != null ?
a.dominantUnmodeledScript : "NONE";
return new FeatureComponents(Float.NaN, Float.NaN, Float.NaN,
- a.z4, a.z5, a.z6, a.z7, a.z8, a.z9, a.logit, "NONE", null,
0);
+ a.z4, a.z5, a.z6, a.z7, a.z8, a.z9, a.logit, script, null,
0);
}
return new FeatureComponents(a.z1, a.z2, a.z3, a.z4, a.z5, a.z6, a.z7,
a.z8, a.z9, a.logit, a.dominantScript, combinerWeights,
diff --git
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnmodeledScriptTest.java
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnmodeledScriptTest.java
new file mode 100644
index 0000000000..42e960dd97
--- /dev/null
+++
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/UnmodeledScriptTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Random;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import org.apache.tika.quality.TextQualityScore;
+
+/**
+ * Real text in a script the bundled model has no usable table for must read as
+ * unmodeled (coverage 0, not junk), and scripts it does model must be
unaffected.
+ */
+public class UnmodeledScriptTest {
+
+ private static JunkDetector detector;
+
+ @BeforeAll
+ static void loadModel() throws Exception {
+ detector = JunkDetector.loadFromClasspath();
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "YI|ꆈꌠꁱꂷ ꀉꂾꑭꇬ ꌋꆀꃅꑍ ꆈꌠꉙ ꀊꁨꌠ ꄉꈌꌠ ꑳꌠꆏ ꃅꐥꇬ ꌠꅇ",
+ "MONGOLIAN|ᠮᠣᠩᠭᠣᠯ ᠤᠯᠤᠰ ᠪᠣᠯ ᠳᠣᠷᠤᠨᠠᠳᠤ ᠠᠽᠢ ᠶᠢᠨ ᠬᠣᠢᠳᠤ ᠬᠡᠰᠡᠭ ᠲᠦ
ᠣᠷᠣᠰᠢᠳᠠᠭ",
+ "CANADIAN_ABORIGINAL|ᐃᓄᒃᑎᑐᑦ ᐅᖃᐅᓯᖅ ᐃᓄᐃᑦ ᐅᖃᐅᓯᖓᑦ ᑲᓇᑕᒥ ᓄᓇᕗᒻᒥ ᐊᒻᒪ
ᓄᓇᕕᒻᒥ",
+ "CHEROKEE|ᏣᎳᎩ ᎦᏬᏂᎯᏍᏗ ᎠᏂᏴᏫᏯ ᎤᏂᎦᏚ ᏚᏂᏃᎮᎴ ᏣᎳᎩ ᎠᏰᎵ ᏓᎵᏍᏓᏴᏗ",
+ "ADLAM|𞤀𞤣𞤤𞤢𞤥 𞤫𞤼𞤵 𞤦𞤭𞤲𞤣𞤭 𞤯𞤫𞤲𞤺𞤢𞤤 𞤆𞤵𞤤𞤢𞤪 𞤲𞤣𞤫𞤪 𞤀𞤬𞤪𞤭𞤳𞤢"
+ })
+ void unmodeledScriptIsNotJunk(String script, String text) {
+ assertFalse(detector.knownScripts().contains(script), script + "
should be unmodeled");
+ TextQualityScore s = detector.score(text);
+ assertTrue(s.isUnmodeled(), "coverage should be 0: " + s);
+ assertEquals(script, s.getDominantScript());
+ assertTrue(s.getZScore() > 0, "coherent unmodeled-script text scored
as junk: " + s);
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "GREEK|Η Ελλάδα είναι χώρα της νοτιοανατολικής Ευρώπης. Άρθρα για
την ιστορία της χώρας.",
+ "LATIN|The quick brown fox jumps over the lazy dog and keeps
running down the road.",
+ "CYRILLIC|Москва является столицей России и крупнейшим городом
страны."
+ })
+ void modeledScriptFullyCovered(String script, String text) {
+ assertTrue(detector.knownScripts().contains(script));
+ TextQualityScore s = detector.score(text);
+ assertEquals(1f, s.getCoverage(), 1e-6f, s.toString());
+ assertEquals(script, s.getDominantScript());
+ assertTrue(s.getZScore() > 0, s.toString());
+ }
+
+ @Test
+ void mixedTextIsPartiallyCovered() {
+ TextQualityScore s = detector.score(
+ "Mongolia ᠮᠣᠩᠭᠣᠯ ᠤᠯᠤᠰ ᠪᠣᠯ ᠳᠣᠷᠤᠨᠠᠳᠤ is a country in East
Asia.");
+ assertEquals("LATIN", s.getDominantScript());
+ assertTrue(s.getCoverage() > 0f && s.getCoverage() < 1f, s.toString());
+ }
+
+ @Test
+ void unmodeledScriptSaladIsJunk() {
+ int[] pool = "ꆈꌠꁱꂷᠮᠣᠩᠭᐃᓄᒃᑎᏣᎳᎩᎦ".codePoints().toArray();
+ Random r = new Random(20261103L);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 60; i++) {
+ sb.appendCodePoint(pool[r.nextInt(pool.length)]);
+ if (r.nextInt(5) == 0) {
+ sb.append(' ');
+ }
+ }
+ TextQualityScore s = detector.score(sb.toString());
+ assertTrue(s.getZScore() < 0, "script salad should stay junk: " + s);
+ }
+
+ @Test
+ void letterFreeTextDoesNotOutscoreProse() {
+ float prose = detector.score("Москва является столицей России и
крупнейшим городом.")
+ .getZScore();
+ for (String letterFree : new String[]{" \n\n\t ", "╔══╗ ║░░║
╚══╝ ▒▓█ ┌─┐", "2026-11-03 3.14159"}) {
+ TextQualityScore s = detector.score(letterFree);
+ assertTrue(Float.isNaN(s.getCoverage()), s.toString());
+ assertTrue(s.getZScore() < 0 && s.getZScore() < prose,
+ "letter-free text must not look like clean prose: " + s);
+ }
+ }
+}
