This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new aab6323021 TIKA-5044: junk detector's HTML cleaner decodes numeric
character references as browsers do (#3537)
aab6323021 is described below
commit aab632302159543015130e6648240bbd43b45d72
Author: Tim Allison <[email protected]>
AuthorDate: Wed Nov 4 16:31:42 2026 -0500
TIKA-5044: junk detector's HTML cleaner decodes numeric character
references as browsers do (#3537)
HtmlContentCleaner emitted the raw code point of any numeric reference, so
– became a C1 control instead of an en dash, and � / surrogate
references produced NUL and lone surrogates. Follow the WHATWG numeric
character reference rules (0x80-0x9F via windows-1252; NUL, surrogates and
values past U+10FFFF to U+FFFD), and expand all references in one pass so
an
expansion is never re-read as a reference.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 5 ++
.../tika/ml/junkdetect/HtmlContentCleaner.java | 70 ++++++++++++----------
.../tika/ml/junkdetect/HtmlContentCleanerTest.java | 65 ++++++++++++++++++++
.../junkdetect/JunkFilterEncodingDetectorTest.java | 3 -
4 files changed, 109 insertions(+), 34 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index e03b03d1e9..289d382705 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -108,6 +108,11 @@ Release 4.3.0 - ???
becomes <b>Note:</b> ); u and s are unchanged. The Markdown writer does
the same for
**, * and ~~, which it wrote as literal asterisks for <b>Note: </b>
(TIKA-5018).
+ * The junk detector, which arbitrates between candidate charsets, decodes
numeric
+ character references in HTML as browsers do: €-Ÿ are the
windows-1252
+ characters (– is an en dash, not a C1 control), and �, surrogates
and values
+ past U+10FFFF are U+FFFD. One pass, so &#x41; stays A
(TIKA-5044).
+
OTHER CHANGES
* The junk detector no longer scores text in scripts its model has no
usable table
diff --git
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
index e531730bce..5b30663ba4 100644
---
a/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
+++
b/tika-ml/tika-ml-junkdetect/src/main/java/org/apache/tika/ml/junkdetect/HtmlContentCleaner.java
@@ -35,10 +35,15 @@ import org.apache.tika.ml.chardetect.HtmlByteStripper;
*/
public final class HtmlContentCleaner {
- private static final Pattern ENTITY_DEC = Pattern.compile("&#(\\d{1,7});");
- private static final Pattern ENTITY_HEX =
Pattern.compile("&#[xX]([0-9a-fA-F]{1,6});");
- private static final Pattern ENTITY_NAMED =
- Pattern.compile("&(amp|lt|gt|quot|apos|nbsp|copy|reg);");
+ private static final Pattern ENTITY = Pattern.compile(
+
"&(?:#[xX]([0-9a-fA-F]+)|#(\\d+)|(amp|lt|gt|quot|apos|nbsp|copy|reg));");
+
+ /** WHATWG remaps numeric references 0x80-0x9F to these windows-1252
characters; 0 = kept. */
+ private static final char[] C1_REMAP = {
+ '\u20AC', 0, '\u201A', '\u0192', '\u201E', '\u2026', '\u2020',
'\u2021',
+ '\u02C6', '\u2030', '\u0160', '\u2039', '\u0152', 0, '\u017D', 0,
+ 0, '\u2018', '\u2019', '\u201C', '\u201D', '\u2022', '\u2013',
'\u2014',
+ '\u02DC', '\u2122', '\u0161', '\u203A', '\u0153', 0, '\u017E',
'\u0178'};
private HtmlContentCleaner() {
}
@@ -64,34 +69,20 @@ public final class HtmlContentCleaner {
/**
* Expand HTML entities to codepoints. Numeric refs ({@code ©},
- * {@code ©}) are fully decoded; a small set of common named entities
- * is mapped; other named entities pass through literally.
+ * {@code ©}) are decoded as a browser does (WHATWG "numeric character
+ * reference end state"); a small set of common named entities is mapped;
+ * other named entities pass through literally.
*/
static String expandHtmlEntities(String s) {
- s = ENTITY_DEC.matcher(s).replaceAll(mr -> {
- try {
- int cp = Integer.parseInt(mr.group(1));
- if (cp >= 0 && cp <= 0x10FFFF) {
- return Matcher.quoteReplacement(new
String(Character.toChars(cp)));
- }
- } catch (NumberFormatException ignored) {
- // overflow — leave literal
+ // one pass, so an expansion is never re-read as a reference
("&#x41;" stays "A")
+ return ENTITY.matcher(s).replaceAll(mr -> {
+ if (mr.group(1) != null) {
+ return numericReference(mr.group(1), 16);
}
- return Matcher.quoteReplacement(mr.group());
- });
- s = ENTITY_HEX.matcher(s).replaceAll(mr -> {
- try {
- int cp = Integer.parseInt(mr.group(1), 16);
- if (cp >= 0 && cp <= 0x10FFFF) {
- return Matcher.quoteReplacement(new
String(Character.toChars(cp)));
- }
- } catch (NumberFormatException ignored) {
- // overflow — leave literal
+ if (mr.group(2) != null) {
+ return numericReference(mr.group(2), 10);
}
- return Matcher.quoteReplacement(mr.group());
- });
- s = ENTITY_NAMED.matcher(s).replaceAll(mr -> {
- switch (mr.group(1)) {
+ switch (mr.group(3)) {
case "amp": return "&";
case "lt": return "<";
case "gt": return ">";
@@ -99,10 +90,27 @@ public final class HtmlContentCleaner {
case "apos": return "'";
case "nbsp": return " ";
case "copy": return "©";
- case "reg": return "®";
- default: return Matcher.quoteReplacement(mr.group());
+ default: return "®";
}
});
- return s;
+ }
+
+ /** NUL, surrogates and values past U+10FFFF become U+FFFD; 0x80-0x9F map
via windows-1252. */
+ private static String numericReference(String digits, int radix) {
+ int start = 0;
+ while (start < digits.length() - 1 && digits.charAt(start) == '0') {
+ start++;
+ }
+ int cp = -1;
+ if (digits.length() - start <= 7) {
+ cp = Integer.parseInt(digits.substring(start), radix);
+ }
+ if (cp <= 0 || cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF)) {
+ return "\uFFFD";
+ }
+ if (cp >= 0x80 && cp <= 0x9F && C1_REMAP[cp - 0x80] != 0) {
+ return String.valueOf(C1_REMAP[cp - 0x80]);
+ }
+ return Matcher.quoteReplacement(new String(Character.toChars(cp)));
}
}
diff --git
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HtmlContentCleanerTest.java
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HtmlContentCleanerTest.java
new file mode 100644
index 0000000000..83695ada7a
--- /dev/null
+++
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/HtmlContentCleanerTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.ml.junkdetect;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+public class HtmlContentCleanerTest {
+
+ /** Numeric references decode as a browser does (WHATWG numeric character
reference end state). */
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', value = {
+ "–|–",
+ "–|–",
+ "’|’",
+ "€|€",
+ "Ÿ|Ÿ",
+ "|\u0081",
+ "|\u009D",
+ "�|�",
+ "�|�",
+ "�|�",
+ "�|�",
+ "�|�",
+ "�|�",
+ "|�",
+ "A|A",
+ "$|$",
+ "\|\\",
+ "©|©",
+ "|"})
+ public void numericReferences(String in, String expected) {
+ assertEquals(expected, HtmlContentCleaner.expandHtmlEntities(in));
+ }
+
+ @Test
+ public void expansionIsNeverReadAgain() {
+ assertEquals("A",
HtmlContentCleaner.expandHtmlEntities("&#x41;"));
+ assertEquals("&",
HtmlContentCleaner.expandHtmlEntities("&amp;"));
+ assertEquals("©",
HtmlContentCleaner.expandHtmlEntities("&#169;"));
+ }
+
+ @Test
+ public void cleanedPageShowsWhatABrowserShows() {
+ assertEquals("price – €5 “ok”",
+ HtmlContentCleaner.clean("<p>price – €5
“ok”</p>").trim());
+ }
+}
diff --git
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
index f1ae65a856..3c35ec26fa 100644
---
a/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
+++
b/tika-ml/tika-ml-junkdetect/src/test/java/org/apache/tika/ml/junkdetect/JunkFilterEncodingDetectorTest.java
@@ -407,9 +407,6 @@ public class JunkFilterEncodingDetectorTest {
// Unknown named entity → left as literal
assertEquals("&unknown;",
JunkFilterEncodingDetector.expandHtmlEntities("&unknown;"));
- // Out-of-range numeric → left as literal (passes overflow guard)
- assertEquals("�",
- JunkFilterEncodingDetector.expandHtmlEntities("�"));
}
@Test
