This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new c58ba0e53f TIKA-5052: restore the ALL-s7 charset model; report
windows-1252 for identical-decode siblings (#3631)
c58ba0e53f is described below
commit c58ba0e53fd92e09f6defe9d3f9120dc9b6ba74e
Author: Tim Allison <[email protected]>
AuthorDate: Tue Nov 17 11:20:53 2026 -0500
TIKA-5052: restore the ALL-s7 charset model; report windows-1252 for
identical-decode siblings (#3631)
The F-s13 model shipped in #3602 churned single-byte picks on the 1.2M
corpus with no real-text
gains; ALL-s7 (a40d54f6a6) is restored with its provenance and parity
table. When the top pick
is a single-byte charset whose decode of the probe is byte-identical to
windows-1252, the result
is relabeled windows-1252. DecodeEquivalence.buildTable rejects
multi-byte charsets.
Co-Authored-By: Claude Fable 5.1 <[email protected]>
---
CHANGES.txt | 8 +++--
.../tika/ml/chardetect/DecodeEquivalence.java | 9 ++++++
.../ml/chardetect/MojibusterEncodingDetector.java | 23 +++++++++++---
.../org/apache/tika/ml/chardetect/hashed-ngram.hng | Bin 2742509 -> 2742509
bytes
.../tika/ml/chardetect/hashed-ngram.hng.provenance | 11 +++----
.../tika/ml/chardetect/LatinFallbackTest.java | 35 +++++++++++++++++++++
.../tika/ml/chardetect/hashed-ngram-parity.tsv | 24 +++++++-------
7 files changed, 83 insertions(+), 27 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index aa3a3f85bf..55287541b6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -521,9 +521,11 @@ Release 4.3.0 - ???
* Charset detection no longer reads ASCII text with a few control
characters (logs with
escape or form-feed bytes, IRC colour codes, record separators, banners)
as BOM-less
- UTF-16: the model was retrained with such text, and when it still returns
UTF-16 or
- UTF-32 as its only candidate, the junk filter now compares that decode
against
- windows-1252 and keeps whichever reads as text (TIKA-5052).
+ UTF-16: when the model returns UTF-16 or UTF-32 as its only candidate,
the junk filter
+ now compares that decode against windows-1252 and keeps whichever reads
as text. A
+ single-byte label whose decode of the probe is identical to
windows-1252's (one stray
+ accented byte in English labelled windows-1254 or x-MacRoman) is reported
as
+ windows-1252 (TIKA-5052).
* The junk model that arbitrates between charset candidates was retrained
with UTF-16
misreads of every script (text read as UTF-16, and UTF-16 text read as a
single-byte
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/DecodeEquivalence.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/DecodeEquivalence.java
index f194c216ec..7b22b4b25d 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/DecodeEquivalence.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/DecodeEquivalence.java
@@ -111,6 +111,15 @@ public final class DecodeEquivalence {
private static char[] buildTable(Charset cs) {
try {
+ // a lone high byte decodes to one U+FFFD under UTF-8/UTF-16 too,
so the per-byte
+ // check below cannot exclude them on its own
+ try {
+ if (cs.newEncoder().maxBytesPerChar() != 1f) {
+ return null;
+ }
+ } catch (UnsupportedOperationException e) {
+ // decode-only charset: rely on the per-byte check
+ }
CharsetDecoder dec = cs.newDecoder()
.onMalformedInput(CodingErrorAction.REPLACE)
.onUnmappableCharacter(CodingErrorAction.REPLACE)
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
index f67258b541..013ca52a35 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/java/org/apache/tika/ml/chardetect/MojibusterEncodingDetector.java
@@ -556,9 +556,14 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
* <p>Replaces the prior strict gates ({@code countHighBytes < 5}
* AND {@code byteIdenticalOnProbe(top, win-1252)}) which left
* ≥ 5-high-byte Western European pages unprotected.</p>
+ *
+ * <p>First, any single-byte label whose decode of the probe is identical
to
+ * windows-1252's is relabeled: the model is choosing between
indistinguishable
+ * decodes (one stray accented byte in English labeled windows-1254), and
the WHATWG
+ * default is the stable answer.</p>
*/
- private static List<EncodingResult> applyLatinSiblingFallback(byte[] probe,
-
List<EncodingResult> ranked) {
+ static List<EncodingResult> applyLatinSiblingFallback(byte[] probe,
+ List<EncodingResult>
ranked) {
if (ranked.isEmpty()) {
return ranked;
}
@@ -567,6 +572,11 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
if (WIN1252.equals(topName)) {
return ranked;
}
+ Charset win1252 = Charset.forName(WIN1252);
+ if (!isWideUnicode(top.getCharset())
+ && DecodeEquivalence.byteIdenticalOnProbe(probe,
top.getCharset(), win1252)) {
+ return relabelTop(ranked, win1252);
+ }
// Scoped to Western European Latin family only. Central
// European (win-1250 / ISO-8859-2 / IBM852), Baltic (win-1257 /
// ISO-8859-13), Turkish (win-1254), Maltese (ISO-8859-3),
@@ -581,16 +591,19 @@ public class MojibusterEncodingDetector implements
EncodingDetector {
if (!CharsetConfusables.WESTERN_LATIN_FAMILY.contains(topName) ||
"x-MacRoman".equals(topName)) {
return ranked;
}
- Charset win1252 = Charset.forName(WIN1252);
int winLetters = HighByteLetterStats.countCasedHighByteLetters(probe,
win1252);
int topLetters = HighByteLetterStats.countCasedHighByteLetters(probe,
top.getCharset());
// Tie goes to windows-1252 (WHATWG-canonical default).
if (winLetters < topLetters) {
return ranked;
}
+ return relabelTop(ranked, win1252);
+ }
+
+ private static List<EncodingResult> relabelTop(List<EncodingResult>
ranked, Charset cs) {
+ EncodingResult top = ranked.get(0);
List<EncodingResult> out = new java.util.ArrayList<>(ranked.size());
- out.add(new EncodingResult(win1252, top.getConfidence(), WIN1252,
- top.getResultType()));
+ out.add(new EncodingResult(cs, top.getConfidence(), cs.name(),
top.getResultType()));
for (int i = 1; i < ranked.size(); i++) {
out.add(ranked.get(i));
}
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
index b4ab5c2056..4c2617d6aa 100644
Binary files
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
and
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng
differ
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
index 706b09e320..dbae53b482 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/main/resources/org/apache/tika/ml/chardetect/hashed-ngram.hng.provenance
@@ -13,11 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-model_sha256 056fcf927453a6eacdcd4f7e463a6d58e8862184990038f62f8dbe6738e5455e
-model_md5 a536ff669cd1610cb9212774d72ab2a9
+model_sha256 b1a64c01b6404c1dbe9644a054907e4975e9cb527b12c31895612806f72ebeb4
+model_md5 7cd1606fdff904d026678e966c18dab6
model_bytes 2742509
format HNG2 (see tika-ml/tika-ml-chardetect/README.md)
-status utf16neg "F", seed 13, 2026-11-11; datafix "ALL" (2026-11-07) plus
C0-control negatives; gold v1.12
+status datafix "ALL", seed 7, 2026-11-07; trained on gold v1.12 (R-s11 recipe
plus the data fixes below)
+note 2026-11-13: a retrain adding C0-control negatives (F-s13, TIKA-5052
#3602) was reverted after the 1.2M run: ~10k identical-text label flips
(1254/1252/MacRoman) and 20 real-text regressions; lone UTF-16/32 picks are
arbitrated against windows-1252 in MojibusterEncodingDetector instead
Architecture
43 classes: 37 legacy/UTF-8 charsets (IBM420/IBM424 as -ltr/-rtl pairs),
IBM437, binary,
@@ -38,10 +39,6 @@ Training data (nothing from the 1.2M regression corpus)
Vocalized Arabic in windows-1256 / ISO-8859-6;
Spanish/Portuguese/Catalan/Galician Wikisource and Spanish
Wikiquote in windows-1252 (dialogue/literary text with ¡ ¿ º ª); Kurmanji
Wikipedia in windows-1254.
Text with sparse single NULs (<= 2%) labelled by its charset.
- C0 controls: 3% of single-byte/UTF-8 rows (not EBCDIC) get all C0 types
(00-1F except TAB/LF/CR) at
- weights from per-byte document counts (gold v1.12 web train + a 1.2M-corpus
sample; counts only, no
- bytes; floor 5), rate 1e-4..1e-2, inserted only between 7-bit bytes, half of
them with high bytes
- removed; labelled by their charset.
Tier B rows whose label set decodes identically take windows-1252 as a hard
target.
Negatives: generated UTF-32LE/BE rows (own classes); generated binary (NUL
records, deflate, bytecode,
integer arrays, random, executables; the scattered-NUL text family was
removed) plus the gold v1.12 fit binary-verdict documents
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/LatinFallbackTest.java
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/LatinFallbackTest.java
index f1878b2087..9b3f2b67e1 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/LatinFallbackTest.java
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/java/org/apache/tika/ml/chardetect/LatinFallbackTest.java
@@ -16,14 +16,19 @@
*/
package org.apache.tika.ml.chardetect;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
import org.junit.jupiter.api.Test;
+import org.apache.tika.detect.EncodingResult;
+
/**
* Tests for the byte-walk decode-equivalence helper and the narrow
* Latin→windows-1252 fallback semantics. Integration with the detector
@@ -88,4 +93,34 @@ public class LatinFallbackTest {
assertTrue(DecodeEquivalence.byteIdenticalOnProbe(new byte[0],
WIN1257, WIN1252));
assertTrue(DecodeEquivalence.byteIdenticalOnProbe(new byte[0], IBM852,
WIN1252));
}
+
+ @Test
+ public void multiByteCharsetsNeverLookByteIdentical() {
+ // a lone high byte decodes to one U+FFFD under UTF-8 and UTF-16 as
well
+ byte[] probe = new byte[]{'a', (byte) 0x81};
+ assertFalse(DecodeEquivalence.byteIdenticalOnProbe(probe,
StandardCharsets.UTF_8, WIN1252));
+ assertFalse(DecodeEquivalence.byteIdenticalOnProbe(probe,
StandardCharsets.UTF_16LE, WIN1252));
+ assertFalse(DecodeEquivalence.byteIdenticalOnProbe(probe,
Charset.forName("GB18030"), WIN1252));
+ }
+
+ @Test
+ public void identicalDecodeSiblingIsRelabeledWindows1252() {
+ Charset win1254 = Charset.forName("windows-1254");
+ byte[] english = "The caf\u00E9 on the corner opens at
nine.".getBytes(ISO8859_1);
+ List<EncodingResult> ranked = Arrays.asList(
+ new EncodingResult(win1254, 1.0f, "windows-1254",
EncodingResult.ResultType.STATISTICAL),
+ new EncodingResult(WIN1252, 0.4f, "windows-1252",
EncodingResult.ResultType.STATISTICAL));
+ List<EncodingResult> out =
MojibusterEncodingDetector.applyLatinSiblingFallback(english, ranked);
+ assertEquals(WIN1252, out.get(0).getCharset());
+ assertEquals(1.0f, out.get(0).getConfidence(), 0);
+ assertEquals(2, out.size());
+ // 0xF0 is \u011F in windows-1254 and \u00F0 in windows-1252: a real
Turkish byte keeps its label
+ byte[] turkish = "Do\u011Fu".getBytes(win1254);
+ assertEquals(win1254,
MojibusterEncodingDetector.applyLatinSiblingFallback(turkish, ranked)
+ .get(0).getCharset());
+ List<EncodingResult> cjk = Arrays.asList(
+ new EncodingResult(Charset.forName("GB18030"), 1.0f,
"GB18030", EncodingResult.ResultType.STATISTICAL));
+ assertEquals("GB18030",
MojibusterEncodingDetector.applyLatinSiblingFallback(english, cjk)
+ .get(0).getCharset().name());
+ }
}
diff --git
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
index ad3f576b79..f0a107191a 100644
---
a/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
+++
b/tika-encoding-detectors/tika-encoding-detector-mojibuster/src/test/resources/org/apache/tika/ml/chardetect/hashed-ngram-parity.tsv
@@ -14,16 +14,16 @@
# limitations under the License.
# Parity fixture for HashedNgramModelParityTest: probe name, probe bytes
(hex), reference logits.
# Regenerate with the reference scorer whenever the model changes.
-model-md5 a536ff669cd1610cb9212774d72ab2a9
+model-md5 7cd1606fdff904d026678e966c18dab6
ascii-abstain
506c61696e204153434949206f6e6c792c206e6f7468696e6720746f2073636f72652e0d0a
abstain
-windows-1252
4c6520636166e920657374207472e87320626f6e2c206d657263692062656175636f757020706f757220766f747265206163637565696c20e02050617269732e
-3.728935,-5.657221,-8.863847,-11.482648,-9.561532,-8.604757,-7.43752,-7.47787,-11.954477,-0.52729106,-3.5210848,-6.8066764,-5.745957,0.95063055,-2.6654189,4.4273067,-3.8865416,-9.069194,-3.4056673,-2.6009703,-6.928908,-13.549944,-3.4414327,-1.8783902,1.049321,7.4346766,-0.76980925,9.129718,-2.3844194,-4.8255363,1.1439289,4.446611,-1.823241,-3.9001
[...]
-UTF-8
4772c3bcc39f6520617573204dc3bc6e6368656e20e2809420736368c3b66e65204772c3bcc39f652c20e697a5e69cace8aa9e2e
-2.4913938,-13.793381,-1.4572061,-14.358084,-9.161243,-9.23574,-17.909801,-18.012604,-10.497832,-14.088515,-5.1198583,-0.6564211,-1.7893723,-15.199613,-8.216926,-13.226288,-7.448959,-15.409586,-17.54502,-0.383887,-8.473598,-1.7661183,14.962193,-8.954592,-12.808847,-19.34087,-20.458302,-14.473593,-18.670933,-20.235914,-11.93831,-9.55483,-4.0152555,-10.492821,-11.533362,-3.9111333
[...]
-Shift_JIS
93fa967b8cea82cc8365834c8358836782c582b7814282b182ea82cd83658358836782c582b78142
2.57346,-15.059127,4.1472445,4.398974,3.2613506,2.9596803,-1.2703773,-1.1718445,5.013065,-11.654519,-1.2685089,-10.18289,1.9499466,-18.974812,-4.3752637,-12.470514,-6.9975033,-22.48132,-12.230818,-2.341249,-11.824933,23.784927,-0.3636093,-10.2057905,-23.064056,-22.359735,-13.014805,-28.699781,-6.697913,-16.720522,-17.511112,-25.210554,-15.232973,-20.280565,-5.8799047,3.1145117,-5.4519334,-17.457754
[...]
-ISO-8859-7 b6f1e8f1e120e3e9e120f4e7ed20f0f1fcebe7f8e720f4e7f220e2dfe1f22e
-4.2260146,-7.166552,4.272812,-3.6654096,-17.119062,-17.21886,-10.04226,-10.850519,-1.8220028,-11.028378,-14.251019,-1.2311823,-11.025501,-15.914036,-11.57087,-16.756527,-1.6853925,-6.3898015,14.812312,-1.487704,-0.98001575,-9.570157,-18.158705,-17.645805,-6.141889,-8.478981,10.022151,-18.13142,-1.2372838,-2.2352679,-21.749323,-8.896761,-8.972828,3.966666,-5.2044406,1.3768202,-8.522958,-22.16721,-3.6683455,-5.35420
[...]
-windows-1251
cdeee2e0ff20eff0eee3f0e0ecece020efee20eff0eef4e8ebe0eaf2e8eae520ede0f1e8ebe8ff2e
-4.3219924,-2.6516016,-9.340967,-12.152876,-13.966185,-13.472875,-13.55826,-13.810733,-13.358068,-11.727482,-21.428669,-3.2278848,-5.659675,-1.6961,-7.5128765,-10.297179,-1.32963,-0.46188512,-5.089943,-0.10025105,0.9076651,-25.003895,-7.767557,-5.3756065,22.452518,-8.460794,-2.3472595,-11.188715,-2.718852,-2.2678545,-13.817303,-3.5131617,-3.0150795,0.7813312,-22.011662,3.3440535,-3.1379137,-14.3
[...]
-IBM500
c38583894085a2a340a49540a385a7a38540859540c5c2c3c4c9c36b4081a58583408485a2408183838595a3a27a4051404440484b
-1.7550082,-4.214464,-0.5797895,16.546648,7.4590564,7.533397,10.255611,10.165785,16.111078,-10.486634,-10.018218,-6.621841,-0.5470758,-19.307821,-19.399403,-12.256375,-10.52631,-13.840994,-9.651697,-3.8074477,-1.8314303,0.8553792,-4.122628,-16.783268,-6.5785995,-11.44013,-6.879944,-20.1175,-8.461582,-1.1915878,-25.546877,-17.379385,-10.510866,-6.510443,-8.374821,0.7822373,-6.
[...]
-UTF-16LE-emoji
53006d0069006c0069006e0067002000660061006300650020003dd800de2c0020006e0075006d00620065007200200031002c0030007800300039002c002a000d000a00
-15.549664,-24.161572,-9.073898,-17.394,-16.21087,-15.716525,-17.243904,-17.603678,-14.58064,-8.127118,-1.9639423,-3.2922075,-10.522701,-12.247312,-4.403795,-14.619513,-3.590005,-10.321466,-9.084791,-1.8683473,-4.0146136,-13.013371,-10.761813,-4.666911,-3.0703082,-10.834128,-2.3034816,-8.052512,-1.7130973,-13.0478945,-7.2918253,-8.470657,
[...]
-UTF-16BE-thai 0e2a0e270e310e2a0e140e350e040e230e310e1a
-8.29257,-16.64819,-4.35965,-6.4487634,-7.919255,-8.070051,-16.993357,-16.577213,-6.2294827,-5.588318,0.15071312,-4.5574737,-8.368986,-11.317027,-3.2231424,-7.0018835,-2.9365144,-18.20875,-9.931014,-1.5361279,-4.716726,-7.872422,-2.8873098,-5.687302,-12.370205,-7.995459,-10.747399,-11.426267,-9.6712055,-13.869811,-10.653317,-12.601955,-7.9654365,-8.82498,-6.027836,-9.065274,-13.249044,-7.864153,6.855218,0.112247616,13.834446,-15.1515
[...]
-UTF-32LE
48000000650000006c0000006c0000006f00000020000000770000006f000000720000006c000000640000002000000000f60100
-15.230365,-16.029194,-9.992692,-14.304272,-18.323326,-17.793623,-13.888347,-13.490578,-11.84498,-2.6002812,-5.9193563,-5.3789177,-18.504791,-16.726011,-22.108696,-15.171628,-10.669345,-21.244684,-7.5246096,-5.7280064,-12.366646,-25.123335,-17.600935,-19.455034,-8.627289,-3.2324917,-1.897337,-3.4269407,-9.871976,-18.790285,-10.662663,-2.746646,-13.5467205,1.89393,-12.476049,-
[...]
-UTF-16LE-long
4c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f0020007000610073007300200074006800650020
[...]
-random
2dbf9f901511db1b9499ddf6b72b5c3544cadbf085b0a22369a741858d27322dc88a3686eb481d5d30a2528592acb75beeed34672f90906a585185ab9bfaa434a38cded677964e3a9c3cd417e59921a304e491d98507bab2995b7f66121591971321ff775c9e026ca0aa5ee871cfe23fa67357f22aafc210cdb80826827b12d5a21c2a6461ebd0ed58838b9bb8f55aa769ab474b26c427c92a32878780877582901e974cef0ef669b3895fcd82d65b30d5db4fa866d00257fa2532702604a3db50b3c67e9451c85ef5d28c9059c1ea2ee7612b2f3be3480c1a96c02a905a43250647774d82f3e16e353d63fc0c001aac9041a
[...]
+windows-1252
4c6520636166e920657374207472e87320626f6e2c206d657263692062656175636f757020706f757220766f747265206163637565696c20e02050617269732e
-6.26243,-8.276049,-7.5950513,-15.685164,-18.508417,-18.136816,-11.677787,-11.436772,-11.429723,-1.6022514,-0.6278504,-2.9343238,-3.2420566,4.612255,-1.5458668,3.170168,-2.1325564,0.52321064,-1.605255,-4.855722,-6.62645,-14.765057,-5.6225595,-1.4943169,-2.3204155,7.4401264,-2.5516288,6.9778447,-0.6250175,-4.572941,-0.37654838,1.202957,-0.26864636,-
[...]
+UTF-8
4772c3bcc39f6520617573204dc3bc6e6368656e20e2809420736368c3b66e65204772c3bcc39f652c20e697a5e69cace8aa9e2e
-0.5827861,-6.6113305,0.045845836,-9.239936,-5.9060984,-5.5219054,-9.469475,-9.633721,-8.441814,-7.54745,-13.777663,-4.715149,-3.4075067,-15.77235,-16.508482,-18.398487,-12.1268635,-15.299498,-14.69563,0.13925156,-9.128676,-5.489176,14.18173,-10.594682,-4.7317257,-15.118749,-12.118952,-22.418028,-21.3163,-10.658545,-15.678664,-10.982281,-4.5749016,-1.7791243,-8.110289,-4.906046,
[...]
+Shift_JIS
93fa967b8cea82cc8365834c8358836782c582b7814282b182ea82cd83658358836782c582b78142
-7.482871,-6.7766385,4.062569,3.0263946,-7.778361,-7.413558,-3.5662513,-3.9060385,0.2243073,-13.015116,-18.522057,-14.260608,-3.560188,-18.54181,-17.02574,-40.14331,-13.6332655,-19.063444,-9.753614,-0.43981814,-7.344433,20.272371,-3.1040049,-1.7563919,-3.9684286,-15.076569,4.173208,-24.263617,-6.0777817,-9.948965,-29.124353,-13.09328,0.19826484,2.1880326,-17.013842,-5.945684,-19.390018,-12.687593,4
[...]
+ISO-8859-7 b6f1e8f1e120e3e9e120f4e7ed20f0f1fcebe7f8e720f4e7f220e2dfe1f22e
-0.8203472,-7.461944,-1.9833272,-2.4042392,-7.2492504,-7.193231,-8.779393,-9.237593,1.1447276,-15.360343,-27.163002,-3.4032931,-5.120804,-11.597296,-6.46171,-13.295239,-2.9843912,-4.196037,16.045206,-3.2323692,0.98040044,-2.3271854,-10.323003,-10.868758,-4.4015923,-11.062085,10.325654,-13.378147,-3.8846247,-4.175025,-19.496069,-7.166667,-6.739304,0.17751911,-13.286078,-5.898166,-4.800658,-19.094826,-2.1375165,-3.95
[...]
+windows-1251
cdeee2e0ff20eff0eee3f0e0ecece020efee20eff0eef4e8ebe0eaf2e8eae520ede0f1e8ebe8ff2e
-12.438238,-10.242093,-1.2998546,-5.962235,-13.427248,-12.878796,-16.230131,-17.201805,-1.0009803,-24.018206,-23.622522,-5.2191486,-4.1623316,-11.167844,-13.7162,-26.255186,-2.1109056,0.028141916,-3.6518037,1.7760476,-7.4237423,-13.722433,-3.9797523,-12.691779,21.79608,-13.254122,-0.24368778,-16.654087,-0.72549474,-7.4011917,-5.029045,-7.342651,-3.902788,-2.1215475,-17.628601,4.9885983,-9.793963
[...]
+IBM500
c38583894085a2a340a49540a385a7a38540859540c5c2c3c4c9c36b4081a58583408485a2408183838595a3a27a4051404440484b
-7.6649594,-5.1355267,-0.059511315,15.283144,6.6421494,6.853642,10.779206,10.726931,14.810395,-10.952078,-24.543068,-11.163412,0.65159315,-14.676382,-17.094334,-29.665552,-10.920232,-19.944944,-10.267533,1.8090932,-8.857017,5.5395036,0.2454361,-10.220937,-3.9662151,-13.020994,-1.419175,-23.684883,-2.9006724,-22.812782,-24.694105,-11.838569,-5.818771,-2.858652,-14.0329075,-2.4
[...]
+UTF-16LE-emoji
53006d0069006c0069006e0067002000660061006300650020003dd800de2c0020006e0075006d00620065007200200031002c0030007800300039002c002a000d000a00
-15.003139,-17.027376,-3.51826,-5.1978855,-10.307344,-10.29958,-16.53273,-16.69455,-3.9325695,-9.80203,-23.2849,0.0851226,-7.799486,-19.536577,-15.215047,-18.650007,-9.40576,-9.536814,-7.5072374,-3.7153225,-5.7540827,-12.313613,-8.299613,-12.684108,-5.6328754,-11.970228,-2.3121173,-12.124944,-4.9368258,-3.026019,-13.056631,-10.524815,-13.
[...]
+UTF-16BE-thai 0e2a0e270e310e2a0e140e350e040e230e310e1a
-11.964084,-20.762892,-3.1328537,-18.356253,-8.522627,-8.178322,-15.8935585,-16.665577,-19.966204,-11.75348,-4.8390594,-7.3139324,-13.296401,-18.660704,-11.799979,-15.680888,-11.938468,-4.528831,-15.936991,-7.375014,-5.5594296,-17.357899,-4.5168223,-5.2730837,-2.0711021,-9.265343,-7.2035103,-11.161968,-15.081558,2.4231293,-3.8285866,-6.273818,-14.419546,-13.408076,-6.7828083,-5.356897,-13.405363,-12.566665,8.712523,3.4173558,23.47274
[...]
+UTF-32LE
48000000650000006c0000006c0000006f00000020000000770000006f000000720000006c000000640000002000000000f60100
-17.05875,-14.528189,-15.666076,-0.61441404,-7.322275,-7.337266,-11.801676,-11.66604,3.42458,-15.463148,-25.011389,-3.5088234,-0.9584222,-26.449154,-28.674969,-18.125229,-9.030397,-15.152163,-14.510708,1.0818806,-3.7085783,-14.689876,-8.415785,-23.006014,-11.733483,-11.460316,-8.38566,-24.200026,-0.6148938,-12.609276,-14.809252,-22.7466,-22.387167,-9.81124,-7.145131,-5.720913
[...]
+UTF-16LE-long
4c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f00200070006100730073002000740068006500200035003100320020006200790074006500200063006f006c0075006d006e002000770069006e0064006f0077002e0020004c006f006e006700200065006e006f00750067006800200074006f0020007000610073007300200074006800650020
[...]
+random
2dbf9f901511db1b9499ddf6b72b5c3544cadbf085b0a22369a741858d27322dc88a3686eb481d5d30a2528592acb75beeed34672f90906a585185ab9bfaa434a38cded677964e3a9c3cd417e59921a304e491d98507bab2995b7f66121591971321ff775c9e026ca0aa5ee871cfe23fa67357f22aafc210cdb80826827b12d5a21c2a6461ebd0ed58838b9bb8f55aa769ab474b26c427c92a32878780877582901e974cef0ef669b3895fcd82d65b30d5db4fa866d00257fa2532702604a3db50b3c67e9451c85ef5d28c9059c1ea2ee7612b2f3be3480c1a96c02a905a43250647774d82f3e16e353d63fc0c001aac9041a
[...]