This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4875 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 7d02b1f1a268a8c619241c4788c2147b02be3883 Author: tallison <[email protected]> AuthorDate: Thu Sep 3 05:36:38 2026 -0400 TIKA-4875: improve tika-eval performance --- CHANGES.txt | 7 + .../tika/eval/app/ExtractComparerRunner.java | 4 +- .../apache/tika/eval/app/ExtractProfileRunner.java | 4 +- .../org/apache/tika/eval/app/StatusReporter.java | 18 ++- .../charsoup/core/CharSoupFeatureExtractor.java | 151 ++++++++++++++++++--- .../core/CharSoupFeatureExtractorTest.java | 86 ++++++++++++ .../tika/langdetect/opennlp/OpenNLPDetector.java | 6 +- 7 files changed, 248 insertions(+), 28 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0d7c4604cd..f37fb7e5c5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,12 @@ Release 4.1.0 - unreleased + * tika-eval Profile/Compare speedups: single-pass URL/mail stripping + replaces the bounded regexes in langdetect preprocessing (same output, + 17-290x faster on web text), the default H2 db URL drops MVStore chunk + retention and raises the page cache, and the status log adds a + last-interval docs-per-sec rate next to the cumulative average + (TIKA-4875). + * AVIF images are parsed rather than only detected: HeifParser accepts image/avif, which is the same ISO-BMFF container, so dimensions, EXIF and XMP come out of it the way they do for HEIC (TIKA-4870). diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java index 6a2c2fdea5..191839ac64 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java @@ -183,8 +183,8 @@ public class ExtractComparerRunner { } //default to h2 Path p = Paths.get(dbPath); - return "jdbc:h2:file:" + p.toAbsolutePath(); - + // drop the 45s MVStore chunk retention (bloat + growing compaction cost); CACHE_SIZE is KB + return "jdbc:h2:file:" + p.toAbsolutePath() + ";RETENTION_TIME=0;CACHE_SIZE=1048576"; } private static void execute(Path inputDir, Path extractsA, Path extractsB, String dbPath, EvalConfig evalConfig, PipesReport pipesReportA, diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java index daa2ddc81d..ba76f484a5 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java @@ -124,8 +124,8 @@ public class ExtractProfileRunner { } //default to h2 Path p = Paths.get(dbPath); - return "jdbc:h2:file:" + p.toAbsolutePath(); - + // drop the 45s MVStore chunk retention (bloat + growing compaction cost); CACHE_SIZE is KB + return "jdbc:h2:file:" + p.toAbsolutePath() + ";RETENTION_TIME=0;CACHE_SIZE=1048576"; } private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig, PipesReport pipesReport, diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java index 3f81f17758..484f327fc8 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/StatusReporter.java @@ -38,6 +38,8 @@ public class StatusReporter implements Callable<Long> { private final AtomicBoolean crawlerIsActive; private final long start; private final NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ROOT); + private int lastCnt = 0; + private long lastReportMillis; public StatusReporter(CallablePipesIterator pipesIterator, AtomicInteger filesProcessed, AtomicInteger activeWorkers, AtomicBoolean crawlerIsActive) { @@ -46,6 +48,7 @@ public class StatusReporter implements Callable<Long> { this.activeWorkers = activeWorkers; this.crawlerIsActive = crawlerIsActive; this.start = System.currentTimeMillis(); + this.lastReportMillis = this.start; } @Override @@ -68,13 +71,22 @@ public class StatusReporter implements Callable<Long> { } private void report() { + long now = System.currentTimeMillis(); int cnt = filesProcessed.get(); - long elapsed = System.currentTimeMillis() - start; + long elapsed = now - start; double elapsedSecs = (double) elapsed / (double) 1000; int avg = (elapsedSecs > 5 || cnt > 100) ? (int) ((double) cnt / elapsedSecs) : -1; - String elapsedString = DurationFormatUtils.formatMillis(System.currentTimeMillis() - start); - String docsPerSec = avg > -1 ? String.format(Locale.ROOT, " (%s docs per sec)", numberFormat.format(avg)) : ""; + // the cumulative average declines by construction and masks cliffs + double windowSecs = (double) (now - lastReportMillis) / (double) 1000; + int windowRate = windowSecs > 0 ? (int) ((double) (cnt - lastCnt) / windowSecs) : avg; + lastCnt = cnt; + lastReportMillis = now; + + String elapsedString = DurationFormatUtils.formatMillis(elapsed); + String docsPerSec = avg > -1 ? + String.format(Locale.ROOT, " (%s docs per sec overall; %s in the last interval)", + numberFormat.format(avg), numberFormat.format(windowRate)) : ""; String msg = String.format(Locale.ROOT, "Processed %s documents in %s%s.", numberFormat.format(cnt), elapsedString, docsPerSec); LOGGER.info(msg); diff --git a/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java b/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java index faae8cb85b..dd02bf273d 100644 --- a/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java +++ b/tika-langdetect/tika-langdetect-charsoup-core/src/main/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractor.java @@ -17,7 +17,6 @@ package org.apache.tika.langdetect.charsoup.core; import java.text.Normalizer; -import java.util.regex.Pattern; /** * Extracts character n-gram features from text using the hashing trick (FNV-1a). @@ -36,7 +35,7 @@ import java.util.regex.Pattern; * <h3>Pipeline</h3> * <ol> * <li>Truncate input at {@link #MAX_TEXT_LENGTH} chars</li> - * <li>Strip URLs and emails (TIKA-2777 bounded patterns)</li> + * <li>Strip URLs and emails</li> * <li>NFC normalize</li> * <li>Iterate codepoints (surrogate-safe)</li> * <li>Skip transparent characters (see {@link #isTransparent(int)})</li> @@ -73,11 +72,26 @@ public class CharSoupFeatureExtractor { /** Underscore sentinel codepoint used for word boundary bigrams. */ static final int SENTINEL = '_'; - // TIKA-2777: bounded regexes to avoid catastrophic backtracking - private static final Pattern URL_REGEX = - Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}"); - private static final Pattern MAIL_REGEX = - Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}"); + // char classes for the URL/mail scanners below (TIKA-4875) + private static final boolean[] URL_CHARS = new boolean[128]; + private static final boolean[] MAIL_CHARS = new boolean[128]; // domain head: no '.' + private static final boolean[] MAIL_DOT_CHARS = new boolean[128]; // local part / domain tail + + static { + for (int c = '0'; c <= '9'; c++) { + URL_CHARS[c] = MAIL_CHARS[c] = MAIL_DOT_CHARS[c] = true; + } + for (int c = 'A'; c <= 'Z'; c++) { + URL_CHARS[c] = MAIL_CHARS[c] = MAIL_DOT_CHARS[c] = true; + int l = c + ('a' - 'A'); + URL_CHARS[l] = MAIL_CHARS[l] = MAIL_DOT_CHARS[l] = true; + } + for (char c : "-_.?&~;+=/#".toCharArray()) { + URL_CHARS[c] = true; + } + MAIL_CHARS['-'] = MAIL_CHARS['_'] = true; + MAIL_DOT_CHARS['-'] = MAIL_DOT_CHARS['_'] = MAIL_DOT_CHARS['.'] = true; + } /** Arabic Tatweel (kashida) — a typographic stretching character (U+0640). */ private static final int TATWEEL = 0x0640; @@ -244,17 +258,9 @@ public class CharSoupFeatureExtractor { * @return cleaned, NFC-normalized text */ public static String preprocessNoTruncate(String rawText) { - // Strip URLs and emails. Both regexes scan the entire input on every call; - // skip each unless its required marker is present ("://" for URL_REGEX, "@" - // for MAIL_REGEX). This is a no-op for the common (markerless) case — the - // output is identical — but avoids a full-buffer regex scan + Matcher alloc. - String text = rawText; - if (text.indexOf("://") >= 0) { - text = URL_REGEX.matcher(text).replaceAll(" "); - } - if (text.indexOf('@') >= 0) { - text = MAIL_REGEX.matcher(text).replaceAll(" "); - } + // order matters: the URL replacement is a barrier ("http://aaaaaaaaaa@bb"; is not a mail match) + String text = stripUrls(rawText); + text = stripEmails(text); // NFC normalize if (!Normalizer.isNormalized(text, Normalizer.Form.NFC)) { @@ -264,6 +270,115 @@ public class CharSoupFeatureExtractor { return text; } + /** + * Equivalent to {@code "https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}"} replaced with " ". + * Matches cannot overlap (':' is outside the char class), so consuming them left to + * right reproduces the regex's leftmost order. + */ + private static String stripUrls(String text) { + int n = text.length(); + StringBuilder sb = null; + int emitted = 0; + int i = 0; + while (i < n) { + if (text.charAt(i) == 'h') { + int afterScheme = -1; + if (text.startsWith("http://";, i)) { + afterScheme = i + 7; + } else if (text.startsWith("https://";, i)) { + afterScheme = i + 8; + } + if (afterScheme > 0) { + int max = Math.min(n, afterScheme + 10000); + int k = afterScheme; + while (k < max && text.charAt(k) < 128 && URL_CHARS[text.charAt(k)]) { + k++; + } + if (k - afterScheme >= 10) { + if (sb == null) { + sb = new StringBuilder(n); + } + sb.append(text, emitted, i).append(' '); + emitted = k; + i = k; + continue; + } + } + } + i++; + } + if (sb == null) { + return text; + } + return sb.append(text, emitted, n).toString(); + } + + /** + * Equivalent to + * {@code "[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}"} replaced with " ". + * Rebuilt around each '@': the local part scans back (max 100, not past the previous + * match, where find() resumes); ahead, the dotless head runs greedily (max 100) and + * yields exactly one char to the tail when the tail cannot otherwise start -- the + * only backtrack the regex can take ("a@bb" matches, "a@b" does not). + * No match spans a second '@', so left to right is leftmost order. + */ + private static String stripEmails(String text) { + int n = text.length(); + StringBuilder sb = null; + int emitted = 0; + int floor = 0; + int i = 0; + while (i < n) { + if (text.charAt(i) == '@') { + int s = i; + while (s > floor && i - s < 100) { + char p = text.charAt(s - 1); + if (p < 128 && MAIL_DOT_CHARS[p]) { + s--; + } else { + break; + } + } + if (s < i) { + int m = i + 1; + int cap2 = Math.min(n, m + 100); + int j = m; + while (j < cap2 && text.charAt(j) < 128 && MAIL_CHARS[text.charAt(j)]) { + j++; + } + int end = -1; + if (j > m) { + if (j < n && text.charAt(j) < 128 && MAIL_DOT_CHARS[text.charAt(j)]) { + int cap3 = Math.min(n, j + 100); + int k = j; + while (k < cap3 && text.charAt(k) < 128 && MAIL_DOT_CHARS[text.charAt(k)]) { + k++; + } + end = k; + } else if (j - m >= 2) { + end = j; + } + } + if (end >= 0) { + if (sb == null) { + sb = new StringBuilder(n); + } + sb.append(text, emitted, s).append(' '); + emitted = end; + floor = end; + i = end; + continue; + } + } + } + i++; + } + if (sb == null) { + return text; + } + return sb.append(text, emitted, n).toString(); + } + /** * Determine whether a codepoint should be treated as transparent (skipped) * during bigram extraction and word tokenization. diff --git a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java index 9a557b8e86..e00570c956 100644 --- a/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java +++ b/tika-langdetect/tika-langdetect-charsoup/src/test/java/org/apache/tika/langdetect/charsoup/core/CharSoupFeatureExtractorTest.java @@ -21,6 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.regex.Pattern; + import org.junit.jupiter.api.Test; @@ -93,6 +98,87 @@ public class CharSoupFeatureExtractorTest { assertArrayEquals(countsC, countsD); } + @Test + public void testUrlMailStrippingMatchesGreedyRegexReference() { + // TIKA-4875: the scanners must stay byte-identical to the regexes they replaced -- + // the langdetect and junkdetect models were trained on this exact preprocessing. + // Inputs are NFC-stable, so preprocessNoTruncate's NFC step is an identity here. + Pattern greedyUrl = Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}"); + Pattern greedyMail = Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}"); + + List<String> cases = new ArrayList<>(); + // middle mail repeat must backtrack to feed the dot-class tail + cases.add("a@bb"); + cases.add("a@b"); + cases.add("[email protected]"); + cases.add("[email protected]"); + cases.add("a@" + "b".repeat(250)); + cases.add("a".repeat(150) + "@x.y"); + cases.add("a".repeat(200) + "@"); + cases.add("local@local@local"); + cases.add("[email protected] a@bb [email protected]"); + cases.add("@@@@@@"); + cases.add("a@a@a@a@a@a@"); + // mail caps: head {1,100} then tail {1,100} over one long run + cases.add("a@" + "b".repeat(100) + "." + "c".repeat(150)); + cases.add("a@" + "b".repeat(99) + "." + "c".repeat(99)); + cases.add("a@" + "b".repeat(300)); + // find() resumes after a match: leftover run chars are not a fresh local part + cases.add("aa@bb cc@dd ee@ff"); + cases.add("a@bb@cc@dd"); + // URL length boundaries: min 10 after scheme, cap 10000 + cases.add("http://"; + "a".repeat(9)); + cases.add("http://"; + "a".repeat(10)); + cases.add("https://"; + "a".repeat(10000)); + cases.add("https://"; + "a".repeat(10001)); + cases.add("https://"; + "a".repeat(10005) + "@bb"); + cases.add("http://http://aaaaaaaaaa";); + cases.add("http://aaaahttp://bbbbbbbbbb";); + cases.add("hhttp://aaaaaaaaaaa";); + cases.add("http:/notaurl http//nope https:/x"); + // the URL pass runs first; its replacement is a barrier for the mail pass + cases.add("http://aaaaaaaaaa@bb";); + cases.add("a@http://aaaaaaaaaa";); + cases.add("[email protected]"); + cases.add("[email protected]/http://foobarbazqux";); + // non-ASCII neighbors exercise the < 128 guards + cases.add("é@bb"); + cases.add("aé@bb"); + cases.add("a@büc.d"); + cases.add("http://aéaaaaaaaaaa";); + cases.add("see http://example.com/a/b?q=1#f and mail bob.smith@sub-domain_x.example.org."); + + long seed = new Random().nextLong(); + Random random = new Random(seed); + String[] atoms = {"a", "B", "9", ".", "-", "_", "@", ":", "/", "#", "?", "=", " ", + "http://";, "https://";, "http", "://", "@a.", "é", "aaaaaaaaaa"}; + for (int i = 0; i < 5000; i++) { + int len = 1 + random.nextInt(60); + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < len; j++) { + sb.append(atoms[random.nextInt(atoms.length)]); + } + cases.add(sb.toString()); + } + // long-run shapes that hit the 100/10000 caps + for (int i = 0; i < 50; i++) { + StringBuilder sb = new StringBuilder(); + while (sb.length() < 3000) { + sb.append("a".repeat(1 + random.nextInt(400))); + sb.append(atoms[random.nextInt(atoms.length)]); + } + cases.add(sb.toString()); + } + + for (String text : cases) { + String expected = greedyMail + .matcher(greedyUrl.matcher(text).replaceAll(" ")) + .replaceAll(" "); + String actual = CharSoupFeatureExtractor.preprocessNoTruncate(text); + assertEquals(expected, actual, "seed=" + seed + " input=" + text); + } + } + @Test public void testURLStripping() { CharSoupFeatureExtractor ext = new CharSoupFeatureExtractor(NUM_BUCKETS); diff --git a/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java b/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java index 243976104c..5bae71b4d4 100644 --- a/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java +++ b/tika-langdetect/tika-langdetect-opennlp/src/main/java/org/apache/tika/langdetect/opennlp/OpenNLPDetector.java @@ -220,11 +220,11 @@ public class OpenNLPDetector extends LanguageDetector { private static class TikaUrlCharSequenceNormalizer implements CharSequenceNormalizer { //use this custom copy/paste of opennlp to avoid long, long hang with mail_regex - //TIKA-2777 + //TIKA-2777. TIKA-4875: possessive where match-equivalent; the middle repeat must stay greedy ("a@bb") private static final Pattern URL_REGEX = - Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}"); + Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}+"); private static final Pattern MAIL_REGEX = - Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}"); + Pattern.compile("[-_.0-9A-Za-z]{1,100}+@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}+"); private static final TikaUrlCharSequenceNormalizer INSTANCE = new TikaUrlCharSequenceNormalizer();
