This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4868-performance-improvements in repository https://gitbox.apache.org/repos/asf/tika.git
commit 953b23251a3606958d1657544bdf802962074ef8 Author: tallison <[email protected]> AuthorDate: Wed Sep 2 06:28:57 2026 -0400 TIKA-4868: single-pass text cleanup in WordExtractor and markdown handler --- CHANGES.txt | 6 +++ .../apache/tika/sax/ToMarkdownContentHandler.java | 13 ++++- .../tika/parser/microsoft/WordExtractor.java | 56 +++++++++++++++++----- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d5beba927b..dee363d60b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,11 @@ Release 4.1.0 - unreleased + * WordExtractor (.doc) cleans each character run in one pass instead of + four chained replace/replaceAll copies, and tests paragraph blankness + without a regex replaceAll; ToMarkdownContentHandler collapses line + breaks in one pass with no copy for clean runs. ~39% off parsing a + text-heavy 2MB .doc (TIKA-4868). + * tika-server's raw-output endpoints (/tika, /tika/text, ...) carry the extracted content as raw UTF-8 bytes from the pipes worker to the HTTP response instead of a Smile-encoded string: one encode in the worker diff --git a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java index 5903a6d818..0c28ef40c2 100644 --- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java +++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java @@ -659,7 +659,18 @@ public class ToMarkdownContentHandler extends DefaultHandler { } private static String collapseLineBreaks(String s) { - return s.replace('\r', ' ').replace('\n', ' '); + // single pass, and no copy at all for the common clean run + char[] chars = null; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '\r' || c == '\n') { + if (chars == null) { + chars = s.toCharArray(); + } + chars[i] = ' '; + } + } + return chars == null ? s : new String(chars); } private static String withTrailingNewline(String s) { diff --git a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java index 09a2b82767..25ad90d49f 100644 --- a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java +++ b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/WordExtractor.java @@ -110,6 +110,42 @@ public class WordExtractor extends AbstractPOIFSExtractor { * Given a style name, return what tag should be used, and * what style should be applied to it. */ + // matches the old regex [\r\n\s]+ ( \s == [ \t\n\x0B\f\r] ), without the + // full replaceAll copy it used to make per paragraph + private static boolean isBlankParagraph(String text) { + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c != '\r' && c != '\n' && c != ' ' && c != '\t' && c != 0x0B && c != '\f') { + return false; + } + } + return true; + } + + private static String cleanControlCharacters(String text) { + char[] chars = null; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + char r; + if (c == 30) { + r = UNICODECHAR_NONBREAKING_HYPHEN; + } else if (c == 31) { + r = UNICODECHAR_ZERO_WIDTH_SPACE; + } else if (c < 0x20) { + r = '\n'; + } else { + r = c; + } + if (r != c) { + if (chars == null) { + chars = text.toCharArray(); + } + chars[i] = r; + } + } + return chars == null ? text : new String(chars); + } + public static TagAndStyle buildParagraphTagAndStyle(String styleName, boolean isTable) { if (styleName == null || styleName.length() < 2) { @@ -354,7 +390,7 @@ public class WordExtractor extends AbstractPOIFSExtractor { } String text = p.text(); - if (text.replaceAll("[\\r\\n\\s]+", "").isEmpty()) { + if (isBlankParagraph(text)) { // Skip empty paragraphs return 0; } @@ -456,22 +492,16 @@ public class WordExtractor extends AbstractPOIFSExtractor { // Clean up the text String text = cr.text(); - text = text.replace('\r', '\n'); if (text.endsWith("\u0007")) { // Strip the table cell end marker text = text.substring(0, text.length() - 1); } - - // Copied from POI's org/apache/poi/hwpf/converter/AbstractWordConverter.processCharacters: - - // Non-breaking hyphens are returned as char 30 - text = text.replace((char) 30, UNICODECHAR_NONBREAKING_HYPHEN); - - // Non-required hyphens to zero-width space - text = text.replace((char) 31, UNICODECHAR_ZERO_WIDTH_SPACE); - - // Control characters as line break - text = text.replaceAll("[\u0000-\u001f]", "\n"); + // One pass over the run replaces what used to be four chained + // replace/replaceAll copies (this runs once per character run): + // \r and every other control char become line breaks, POI's char 30 + // (non-breaking hyphen) and char 31 (non-required hyphen) become + // their Unicode equivalents. + text = cleanControlCharacters(text); xhtml.characters(text); }
