This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 4f0e11bafa TIKA-4875: improve tika-eval performance (#3123)
4f0e11bafa is described below
commit 4f0e11bafacf55fd79e48be7dc83eff3b2baf6e0
Author: Tim Allison <[email protected]>
AuthorDate: Thu Sep 3 15:55:17 2026 -0400
TIKA-4875: improve tika-eval performance (#3123)
---
CHANGES.txt | 8 ++
.../tika/eval/app/ExtractComparerRunner.java | 12 +-
.../apache/tika/eval/app/ExtractProfileRunner.java | 12 +-
.../org/apache/tika/eval/app/StatusReporter.java | 18 ++-
.../java/org/apache/tika/eval/app/db/JDBCUtil.java | 41 ++++++
.../org/apache/tika/eval/app/db/JDBCUtilTest.java | 65 +++++++++
.../charsoup/core/CharSoupFeatureExtractor.java | 151 ++++++++++++++++++---
.../core/CharSoupFeatureExtractorTest.java | 87 ++++++++++++
.../tika/langdetect/opennlp/OpenNLPDetector.java | 6 +-
9 files changed, 354 insertions(+), 46 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index b1791047ec..2e739e0f21 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,13 @@
Release 4.1.0 - unreleased
+ * tika-eval Profile/Compare speedups: single-pass URL/mail stripping
+ replaces the bounded regexes in langdetect preprocessing (same output,
+ 17-290x faster on web text), the default H2 db URL drops MVStore chunk
+ retention and sizes the page cache at a quarter of the heap clamped to
+ [64MB, 1GB] (override with -Dtika.eval.h2.cacheSizeKb=<kb>), and the
+ status log adds a last-interval docs-per-sec rate next to the cumulative
+ average (TIKA-4875).
+
* New "content-enrichers" config list (TIKA-4872): select the OCR engine
("tesseract-ocr-parser", "tess4j-parser", "openai-vlm-parser", ...) by
name instead of by classpath registration of the image/ocr-* pseudo
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
index 6a2c2fdea5..b76abe7688 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java
@@ -142,7 +142,7 @@ public class ExtractComparerRunner {
}
try {
- String jdbcString = getJdbcConnectionString(dbPath);
+ String jdbcString = JDBCUtil.getJdbcConnectionString(dbPath);
Map<String, String> runInfo = RunInfo.evalInfo(args, evalConfig,
inputDir);
execute(inputDir, extractsADir, extractsBDir, jdbcString,
evalConfig, sideA.pipesReport(), sideB.pipesReport(), runInfo, runInfoA,
runInfoB);
@@ -177,16 +177,6 @@ public class ExtractComparerRunner {
return commandLine.hasOption(opt) ?
Paths.get(commandLine.getOptionValue(opt)) : null;
}
- private static String getJdbcConnectionString(String dbPath) {
- if (dbPath.startsWith("jdbc:")) {
- return dbPath;
- }
- //default to h2
- Path p = Paths.get(dbPath);
- return "jdbc:h2:file:" + p.toAbsolutePath();
-
- }
-
private static void execute(Path inputDir, Path extractsA, Path extractsB,
String dbPath, EvalConfig evalConfig, PipesReport pipesReportA,
PipesReport pipesReportB, Map<String, String>
runInfo, Map<String, String> runInfoA, Map<String, String> runInfoB)
throws SQLException, IOException {
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
index daa2ddc81d..3288f145f2 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java
@@ -98,7 +98,7 @@ public class ExtractProfileRunner {
Path extractsDir = commandLine.hasOption('e') ?
Paths.get(commandLine.getOptionValue('e')) : Paths.get(USAGE_FAIL("Must specify
extracts dir: -i"));
Path inputDir = commandLine.hasOption('i') ?
Paths.get(commandLine.getOptionValue('i')) : extractsDir;
String dbPath = commandLine.hasOption('d') ?
commandLine.getOptionValue('d') : USAGE_FAIL("Must specify the db name: -d");
- String jdbcString = getJdbcConnectionString(dbPath);
+ String jdbcString = JDBCUtil.getJdbcConnectionString(dbPath);
if (commandLine.hasOption('n')) {
evalConfig.setNumWorkers(Integer.parseInt(commandLine.getOptionValue('n')));
}
@@ -118,16 +118,6 @@ public class ExtractProfileRunner {
return commandLine.hasOption(opt) ?
Paths.get(commandLine.getOptionValue(opt)) : null;
}
- private static String getJdbcConnectionString(String dbPath) {
- if (dbPath.startsWith("jdbc:")) {
- return dbPath;
- }
- //default to h2
- Path p = Paths.get(dbPath);
- return "jdbc:h2:file:" + p.toAbsolutePath();
-
- }
-
private static void execute(Path inputDir, Path extractsDir, String
dbPath, EvalConfig evalConfig, PipesReport pipesReport,
Map<String, String> runInfo) throws
SQLException, IOException {
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
index 3f81f17758..484f327fc8 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java
@@ -38,6 +38,8 @@ public class StatusReporter implements Callable<Long> {
private final AtomicBoolean crawlerIsActive;
private final long start;
private final NumberFormat numberFormat =
NumberFormat.getNumberInstance(Locale.ROOT);
+ private int lastCnt = 0;
+ private long lastReportMillis;
public StatusReporter(CallablePipesIterator pipesIterator, AtomicInteger
filesProcessed, AtomicInteger activeWorkers, AtomicBoolean crawlerIsActive) {
@@ -46,6 +48,7 @@ public class StatusReporter implements Callable<Long> {
this.activeWorkers = activeWorkers;
this.crawlerIsActive = crawlerIsActive;
this.start = System.currentTimeMillis();
+ this.lastReportMillis = this.start;
}
@Override
@@ -68,13 +71,22 @@ public class StatusReporter implements Callable<Long> {
}
private void report() {
+ long now = System.currentTimeMillis();
int cnt = filesProcessed.get();
- long elapsed = System.currentTimeMillis() - start;
+ long elapsed = now - start;
double elapsedSecs = (double) elapsed / (double) 1000;
int avg = (elapsedSecs > 5 || cnt > 100) ? (int) ((double) cnt /
elapsedSecs) : -1;
- String elapsedString =
DurationFormatUtils.formatMillis(System.currentTimeMillis() - start);
- String docsPerSec = avg > -1 ? String.format(Locale.ROOT, " (%s docs
per sec)", numberFormat.format(avg)) : "";
+ // the cumulative average declines by construction and masks cliffs
+ double windowSecs = (double) (now - lastReportMillis) / (double) 1000;
+ int windowRate = windowSecs > 0 ? (int) ((double) (cnt - lastCnt) /
windowSecs) : avg;
+ lastCnt = cnt;
+ lastReportMillis = now;
+
+ String elapsedString = DurationFormatUtils.formatMillis(elapsed);
+ String docsPerSec = avg > -1 ?
+ String.format(Locale.ROOT, " (%s docs per sec overall; %s in
the last interval)",
+ numberFormat.format(avg),
numberFormat.format(windowRate)) : "";
String msg = String.format(Locale.ROOT, "Processed %s documents in
%s%s.", numberFormat.format(cnt), elapsedString, docsPerSec);
LOGGER.info(msg);
diff --git
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/JDBCUtil.java
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/JDBCUtil.java
index 8d0de20e38..43e439722d 100644
---
a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/JDBCUtil.java
+++
b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/JDBCUtil.java
@@ -19,6 +19,8 @@ package org.apache.tika.eval.app.db;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
@@ -40,6 +42,17 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class JDBCUtil {
+
+ /**
+ * Override for the h2 page cache in KB; unclamped, so a big box can go
well
+ * beyond the heap-relative default.
+ */
+ public static final String H2_CACHE_SIZE_KB_PROPERTY =
"tika.eval.h2.cacheSizeKb";
+
+ //h2's own default: 64MB
+ private static final long MIN_H2_CACHE_SIZE_KB = 65_536L;
+ private static final long MAX_H2_CACHE_SIZE_KB = 1_048_576L;
+
private static final Logger LOG = LoggerFactory.getLogger(JDBCUtil.class);
private final String connectionString;
private String driverClass;
@@ -73,6 +86,34 @@ public class JDBCUtil {
}
}
+ /**
+ * If dbPath is already a jdbc string, it is used as is; otherwise this
builds the
+ * tika-eval h2 default: RETENTION_TIME=0 drops the 45s MVStore chunk
retention
+ * (bloat + growing compaction cost) and CACHE_SIZE (KB) is sized by
+ * {@link #getH2CacheSizeKb()}.
+ */
+ public static String getJdbcConnectionString(String dbPath) {
+ if (dbPath.startsWith("jdbc:")) {
+ return dbPath;
+ }
+ Path p = Paths.get(dbPath);
+ return "jdbc:h2:file:" + p.toAbsolutePath() +
";RETENTION_TIME=0;CACHE_SIZE=" + getH2CacheSizeKb();
+ }
+
+ /**
+ * H2's page cache is on heap, so the default is a quarter of the heap
clamped to
+ * [64MB, 1GB] rather than a fixed size that a small JVM cannot afford. Set
+ * {@value #H2_CACHE_SIZE_KB_PROPERTY} to override.
+ */
+ public static long getH2CacheSizeKb() {
+ String override = System.getProperty(H2_CACHE_SIZE_KB_PROPERTY);
+ if (override != null) {
+ return Long.parseLong(override.trim());
+ }
+ long quarterHeapKb = Runtime.getRuntime().maxMemory() / 4 / 1024;
+ return Math.min(MAX_H2_CACHE_SIZE_KB, Math.max(MIN_H2_CACHE_SIZE_KB,
quarterHeapKb));
+ }
+
public static void batchInsert(PreparedStatement insertStatement,
TableInfo table, Map<Cols, String> data) throws SQLException {
try {
diff --git
a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/db/JDBCUtilTest.java
b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/db/JDBCUtilTest.java
new file mode 100644
index 0000000000..79df461426
--- /dev/null
+++
b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/db/JDBCUtilTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.app.db;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class JDBCUtilTest {
+
+ private String originalCacheSize;
+
+ @BeforeEach
+ public void stashCacheSizeProperty() {
+ originalCacheSize =
System.getProperty(JDBCUtil.H2_CACHE_SIZE_KB_PROPERTY);
+ System.clearProperty(JDBCUtil.H2_CACHE_SIZE_KB_PROPERTY);
+ }
+
+ @AfterEach
+ public void restoreCacheSizeProperty() {
+ if (originalCacheSize == null) {
+ System.clearProperty(JDBCUtil.H2_CACHE_SIZE_KB_PROPERTY);
+ } else {
+ System.setProperty(JDBCUtil.H2_CACHE_SIZE_KB_PROPERTY,
originalCacheSize);
+ }
+ }
+
+ @Test
+ public void testJdbcStringPassesThrough() {
+ String jdbc = "jdbc:postgresql://localhost/tika_eval";
+ assertEquals(jdbc, JDBCUtil.getJdbcConnectionString(jdbc));
+ }
+
+ @Test
+ public void testH2Defaults() {
+ long cacheSizeKb = JDBCUtil.getH2CacheSizeKb();
+ assertTrue(cacheSizeKb >= 65_536L && cacheSizeKb <= 1_048_576L,
"clamped to [64MB, 1GB]: " + cacheSizeKb);
+ String connectionString = JDBCUtil.getJdbcConnectionString("mydb");
+ assertTrue(connectionString.startsWith("jdbc:h2:file:"),
connectionString);
+ assertTrue(connectionString.endsWith(";RETENTION_TIME=0;CACHE_SIZE=" +
cacheSizeKb), connectionString);
+ }
+
+ @Test
+ public void testH2CacheSizeOverrideIsUnclamped() {
+ System.setProperty(JDBCUtil.H2_CACHE_SIZE_KB_PROPERTY, "8388608");
+ assertEquals(8_388_608L, JDBCUtil.getH2CacheSizeKb());
+ }
+}
diff --git
a/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
b/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
index faae8cb85b..dd02bf273d 100644
---
a/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
+++
b/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java
@@ -17,7 +17,6 @@
package org.apache.tika.langdetect.charsoup.core;
import java.text.Normalizer;
-import java.util.regex.Pattern;
/**
* Extracts character n-gram features from text using the hashing trick
(FNV-1a).
@@ -36,7 +35,7 @@ import java.util.regex.Pattern;
* <h3>Pipeline</h3>
* <ol>
* <li>Truncate input at {@link #MAX_TEXT_LENGTH} chars</li>
- * <li>Strip URLs and emails (TIKA-2777 bounded patterns)</li>
+ * <li>Strip URLs and emails</li>
* <li>NFC normalize</li>
* <li>Iterate codepoints (surrogate-safe)</li>
* <li>Skip transparent characters (see {@link #isTransparent(int)})</li>
@@ -73,11 +72,26 @@ public class CharSoupFeatureExtractor {
/** Underscore sentinel codepoint used for word boundary bigrams. */
static final int SENTINEL = '_';
- // TIKA-2777: bounded regexes to avoid catastrophic backtracking
- private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
- private static final Pattern MAIL_REGEX =
-
Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+ // char classes for the URL/mail scanners below (TIKA-4875)
+ private static final boolean[] URL_CHARS = new boolean[128];
+ private static final boolean[] MAIL_CHARS = new boolean[128]; //
domain head: no '.'
+ private static final boolean[] MAIL_DOT_CHARS = new boolean[128]; // local
part / domain tail
+
+ static {
+ for (int c = '0'; c <= '9'; c++) {
+ URL_CHARS[c] = MAIL_CHARS[c] = MAIL_DOT_CHARS[c] = true;
+ }
+ for (int c = 'A'; c <= 'Z'; c++) {
+ URL_CHARS[c] = MAIL_CHARS[c] = MAIL_DOT_CHARS[c] = true;
+ int l = c + ('a' - 'A');
+ URL_CHARS[l] = MAIL_CHARS[l] = MAIL_DOT_CHARS[l] = true;
+ }
+ for (char c : "-_.?&~;+=/#".toCharArray()) {
+ URL_CHARS[c] = true;
+ }
+ MAIL_CHARS['-'] = MAIL_CHARS['_'] = true;
+ MAIL_DOT_CHARS['-'] = MAIL_DOT_CHARS['_'] = MAIL_DOT_CHARS['.'] = true;
+ }
/** Arabic Tatweel (kashida) — a typographic stretching character
(U+0640). */
private static final int TATWEEL = 0x0640;
@@ -244,17 +258,9 @@ public class CharSoupFeatureExtractor {
* @return cleaned, NFC-normalized text
*/
public static String preprocessNoTruncate(String rawText) {
- // Strip URLs and emails. Both regexes scan the entire input on every
call;
- // skip each unless its required marker is present ("://" for
URL_REGEX, "@"
- // for MAIL_REGEX). This is a no-op for the common (markerless) case —
the
- // output is identical — but avoids a full-buffer regex scan + Matcher
alloc.
- String text = rawText;
- if (text.indexOf("://") >= 0) {
- text = URL_REGEX.matcher(text).replaceAll(" ");
- }
- if (text.indexOf('@') >= 0) {
- text = MAIL_REGEX.matcher(text).replaceAll(" ");
- }
+ // order matters: the URL replacement is a barrier
("http://aaaaaaaaaa@bb"; is not a mail match)
+ String text = stripUrls(rawText);
+ text = stripEmails(text);
// NFC normalize
if (!Normalizer.isNormalized(text, Normalizer.Form.NFC)) {
@@ -264,6 +270,115 @@ public class CharSoupFeatureExtractor {
return text;
}
+ /**
+ * Equivalent to {@code "https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}"}
replaced with " ".
+ * Matches cannot overlap (':' is outside the char class), so consuming
them left to
+ * right reproduces the regex's leftmost order.
+ */
+ private static String stripUrls(String text) {
+ int n = text.length();
+ StringBuilder sb = null;
+ int emitted = 0;
+ int i = 0;
+ while (i < n) {
+ if (text.charAt(i) == 'h') {
+ int afterScheme = -1;
+ if (text.startsWith("http://";, i)) {
+ afterScheme = i + 7;
+ } else if (text.startsWith("https://";, i)) {
+ afterScheme = i + 8;
+ }
+ if (afterScheme > 0) {
+ int max = Math.min(n, afterScheme + 10000);
+ int k = afterScheme;
+ while (k < max && text.charAt(k) < 128 &&
URL_CHARS[text.charAt(k)]) {
+ k++;
+ }
+ if (k - afterScheme >= 10) {
+ if (sb == null) {
+ sb = new StringBuilder(n);
+ }
+ sb.append(text, emitted, i).append(' ');
+ emitted = k;
+ i = k;
+ continue;
+ }
+ }
+ }
+ i++;
+ }
+ if (sb == null) {
+ return text;
+ }
+ return sb.append(text, emitted, n).toString();
+ }
+
+ /**
+ * Equivalent to
+ * {@code
"[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}"} replaced
with " ".
+ * Rebuilt around each '@': the local part scans back (max 100, not past
the previous
+ * match, where find() resumes); ahead, the dotless head runs greedily
(max 100) and
+ * yields exactly one char to the tail when the tail cannot otherwise
start -- the
+ * only backtrack the regex can take ("a@bb" matches, "a@b" does not).
+ * No match spans a second '@', so left to right is leftmost order.
+ */
+ private static String stripEmails(String text) {
+ int n = text.length();
+ StringBuilder sb = null;
+ int emitted = 0;
+ int floor = 0;
+ int i = 0;
+ while (i < n) {
+ if (text.charAt(i) == '@') {
+ int s = i;
+ while (s > floor && i - s < 100) {
+ char p = text.charAt(s - 1);
+ if (p < 128 && MAIL_DOT_CHARS[p]) {
+ s--;
+ } else {
+ break;
+ }
+ }
+ if (s < i) {
+ int m = i + 1;
+ int cap2 = Math.min(n, m + 100);
+ int j = m;
+ while (j < cap2 && text.charAt(j) < 128 &&
MAIL_CHARS[text.charAt(j)]) {
+ j++;
+ }
+ int end = -1;
+ if (j > m) {
+ if (j < n && text.charAt(j) < 128 &&
MAIL_DOT_CHARS[text.charAt(j)]) {
+ int cap3 = Math.min(n, j + 100);
+ int k = j;
+ while (k < cap3 && text.charAt(k) < 128 &&
MAIL_DOT_CHARS[text.charAt(k)]) {
+ k++;
+ }
+ end = k;
+ } else if (j - m >= 2) {
+ end = j;
+ }
+ }
+ if (end >= 0) {
+ if (sb == null) {
+ sb = new StringBuilder(n);
+ }
+ sb.append(text, emitted, s).append(' ');
+ emitted = end;
+ floor = end;
+ i = end;
+ continue;
+ }
+ }
+ }
+ i++;
+ }
+ if (sb == null) {
+ return text;
+ }
+ return sb.append(text, emitted, n).toString();
+ }
+
/**
* Determine whether a codepoint should be treated as transparent (skipped)
* during bigram extraction and word tokenization.
diff --git
a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
index 9a557b8e86..8d8c3f5f4e 100644
---
a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
+++
b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java
@@ -21,6 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.regex.Pattern;
+
import org.junit.jupiter.api.Test;
@@ -93,6 +98,88 @@ public class CharSoupFeatureExtractorTest {
assertArrayEquals(countsC, countsD);
}
+ @Test
+ public void testUrlMailStrippingMatchesGreedyRegexReference() {
+ // TIKA-4875: the scanners must stay byte-identical to the regexes
they replaced --
+ // the langdetect and junkdetect models were trained on this exact
preprocessing.
+ // Inputs are NFC-stable, so preprocessNoTruncate's NFC step is an
identity here.
+ Pattern greedyUrl =
Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
+ Pattern greedyMail =
Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+
+ List<String> cases = new ArrayList<>();
+ // middle mail repeat must backtrack to feed the dot-class tail
+ cases.add("a@bb");
+ cases.add("a@b");
+ cases.add("[email protected]");
+ cases.add("[email protected]");
+ cases.add("a@" + "b".repeat(250));
+ cases.add("a".repeat(150) + "@x.y");
+ cases.add("a".repeat(200) + "@");
+ cases.add("local@local@local");
+ cases.add("[email protected] a@bb [email protected]");
+ cases.add("@@@@@@");
+ cases.add("a@a@a@a@a@a@");
+ // mail caps: head {1,100} then tail {1,100} over one long run
+ cases.add("a@" + "b".repeat(100) + "." + "c".repeat(150));
+ cases.add("a@" + "b".repeat(99) + "." + "c".repeat(99));
+ cases.add("a@" + "b".repeat(300));
+ // find() resumes after a match: leftover run chars are not a fresh
local part
+ cases.add("aa@bb cc@dd ee@ff");
+ cases.add("a@bb@cc@dd");
+ // URL length boundaries: min 10 after scheme, cap 10000
+ cases.add("http://"; + "a".repeat(9));
+ cases.add("http://"; + "a".repeat(10));
+ cases.add("https://"; + "a".repeat(10000));
+ cases.add("https://"; + "a".repeat(10001));
+ cases.add("https://"; + "a".repeat(10005) + "@bb");
+ cases.add("http://http://aaaaaaaaaa";);
+ cases.add("http://aaaahttp://bbbbbbbbbb";);
+ cases.add("hhttp://aaaaaaaaaaa";);
+ cases.add("http:/notaurl http//nope https:/x");
+ // the URL pass runs first; its replacement is a barrier for the mail
pass
+ cases.add("http://aaaaaaaaaa@bb";);
+ cases.add("a@http://aaaaaaaaaa";);
+ cases.add("[email protected]");
+ cases.add("[email protected]/http://foobarbazqux";);
+ // non-ASCII neighbors exercise the < 128 guards
+ cases.add("é@bb");
+ cases.add("aé@bb");
+ cases.add("a@büc.d");
+ cases.add("http://aéaaaaaaaaaa";);
+ cases.add("see http://example.com/a/b?q=1#f and mail
bob.smith@sub-domain_x.example.org.");
+
+ // random by default so the corpus keeps moving; rerun a failure with
-Dtika.test.seed=<seed>
+ long seed = Long.getLong("tika.test.seed", new Random().nextLong());
+ Random random = new Random(seed);
+ String[] atoms = {"a", "B", "9", ".", "-", "_", "@", ":", "/", "#",
"?", "=", " ",
+ "http://";, "https://";, "http", "://", "@a.", "é",
"aaaaaaaaaa"};
+ for (int i = 0; i < 5000; i++) {
+ int len = 1 + random.nextInt(60);
+ StringBuilder sb = new StringBuilder();
+ for (int j = 0; j < len; j++) {
+ sb.append(atoms[random.nextInt(atoms.length)]);
+ }
+ cases.add(sb.toString());
+ }
+ // long-run shapes that hit the 100/10000 caps
+ for (int i = 0; i < 50; i++) {
+ StringBuilder sb = new StringBuilder();
+ while (sb.length() < 3000) {
+ sb.append("a".repeat(1 + random.nextInt(400)));
+ sb.append(atoms[random.nextInt(atoms.length)]);
+ }
+ cases.add(sb.toString());
+ }
+
+ for (String text : cases) {
+ String expected = greedyMail
+ .matcher(greedyUrl.matcher(text).replaceAll(" "))
+ .replaceAll(" ");
+ String actual =
CharSoupFeatureExtractor.preprocessNoTruncate(text);
+ assertEquals(expected, actual, "-Dtika.test.seed=" + seed + "
input=" + text);
+ }
+ }
+
@Test
public void testURLStripping() {
CharSoupFeatureExtractor ext = new
CharSoupFeatureExtractor(NUM_BUCKETS);
diff --git
a/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
b/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
index 243976104c..5bae71b4d4 100644
---
a/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
+++
b/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java
@@ -220,11 +220,11 @@ public class OpenNLPDetector extends LanguageDetector {
private static class TikaUrlCharSequenceNormalizer implements
CharSequenceNormalizer {
//use this custom copy/paste of opennlp to avoid long, long hang with
mail_regex
- //TIKA-2777
+ //TIKA-2777. TIKA-4875: possessive where match-equivalent; the middle
repeat must stay greedy ("a@bb")
private static final Pattern URL_REGEX =
- Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
+ Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}+");
private static final Pattern MAIL_REGEX =
-
Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+
Pattern.compile("[-_.0-9A-Za-z]{1,100}+@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}+");
private static final TikaUrlCharSequenceNormalizer INSTANCE =
new TikaUrlCharSequenceNormalizer();
