This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4868-performance-improvements in repository https://gitbox.apache.org/repos/asf/tika.git
commit 6ad41e0f200e6b47a7f4aa136ecef67eae55104f Author: tallison <[email protected]> AuthorDate: Tue Sep 1 21:27:33 2026 -0400 improve detector efficiency --- CHANGES.txt | 17 ++++++ .../java/org/apache/tika/detect/MagicDetector.java | 22 ++++++-- .../main/java/org/apache/tika/mime/MagicMatch.java | 19 +++++-- .../main/java/org/apache/tika/mime/MimeTypes.java | 61 ++++++++++++++++------ .../main/java/org/apache/tika/mime/Patterns.java | 14 ++++- .../org/apache/tika/mime/tika-mimetypes.xml | 6 ++- 6 files changed, 112 insertions(+), 27 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index db2f14e11f..8ef82bedf1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,22 @@ Release 4.1.0 - unreleased + * Detection hot-path cleanups: MagicMatch resolves its detector via + double-checked locking instead of a synchronized method per eval; + glob patterns are compiled once at registration instead of per + lookup; MimeTypes.forName reads a ConcurrentHashMap (fixing an + unsynchronized-read race) and indexes the normalized key so + parameterized aliases stop re-locking per call; resource names + containing spaces skip the URI-parse-by-exception; the magic-header + buffer is sized by the stream's known length instead of a fixed + 64KB per detection; and the Adobe Illustrator ranged regex is + gated behind a literal scan (detection results unchanged, + ~28us off every non-AI detection) (TIKA-4868). + + * Magic detection is ~35% faster on unmatched (e.g. plain-text) input: + MagicDetector range scans find first-byte candidates with a tight + scan before running the full masked/case-folded compare, instead of + paying it at every offset in the range (TIKA-4868). + * CSVSniffer reads its detection window once into a shared buffer and runs every delimiter hypothesis against it, instead of re-reading the stream through a pushback/mark-reset stack per delimiter; windows with diff --git a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java index 80a7725453..839b722c3a 100644 --- a/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java +++ b/tika-core/src/main/java/org/apache/tika/detect/MagicDetector.java @@ -490,15 +490,29 @@ public class MagicDetector implements Detector { } } } else { - // Loop until we've covered the entire offset range + // Range scans (e.g. "\nHeader:" over 0:1024) dominate detection time, so + // find first-byte candidates with a tight scan and run the full compare + // only there, instead of paying the masked/case machinery at every offset. + if (length == 0) { + // degenerate empty pattern: preserves the old loop's outcome + return startOffset <= endOffset && startOffset <= buffer.length; + } + int first = pattern[0]; + byte firstMask = mask[0]; for (int i = startOffset; i <= endOffset; i++) { if (i + length > buffer.length) { break; } + int masked0 = buffer[i] & firstMask; + if (this.isStringIgnoreCase) { + masked0 = Character.toLowerCase(masked0); + } + if (masked0 != first) { + continue; + } boolean match = true; - int masked; - for (int j = 0; match && j < length; j++) { - masked = (buffer[i + j] & mask[j]); + for (int j = 1; match && j < length; j++) { + int masked = (buffer[i + j] & mask[j]); if (this.isStringIgnoreCase) { masked = Character.toLowerCase(masked); } diff --git a/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java b/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java index 835d4b90c4..b4ed3a9733 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java +++ b/tika-core/src/main/java/org/apache/tika/mime/MagicMatch.java @@ -33,7 +33,7 @@ class MagicMatch implements Clause { private final String mask; - private MagicDetector detector = null; + private volatile MagicDetector detector = null; MagicMatch(MediaType mediaType, String type, String offset, String value, String mask) { this.mediaType = mediaType; @@ -43,11 +43,20 @@ class MagicMatch implements Clause { this.mask = mask; } - private synchronized MagicDetector getDetector() { - if (detector == null) { - detector = MagicDetector.parse(mediaType, type, offset, value, mask); + private MagicDetector getDetector() { + // Double-checked: eval() runs per magic per detect call, so a synchronized + // method here means thousands of monitor acquisitions per detection. + MagicDetector d = detector; + if (d == null) { + synchronized (this) { + d = detector; + if (d == null) { + d = MagicDetector.parse(mediaType, type, offset, value, mask); + detector = d; + } + } } - return detector; + return d; } public boolean eval(byte[] data) { diff --git a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java index c178729cde..eb0657c3a3 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java +++ b/tika-core/src/main/java/org/apache/tika/mime/MimeTypes.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import javax.xml.namespace.QName; import org.apache.tika.Tika; @@ -103,7 +104,8 @@ public final class MimeTypes implements Detector, Serializable { /** * All the registered MimeTypes indexed on their canonical names */ - private final Map<MediaType, MimeType> types = new HashMap<>(); + // ConcurrentHashMap: forName reads this outside the lock that mutates it + private final Map<MediaType, MimeType> types = new ConcurrentHashMap<>(); /** * The patterns matcher */ @@ -318,11 +320,23 @@ public final class MimeTypes implements Detector, Serializable { * @throws IOException if the stream can not be read */ byte[] readMagicHeader(InputStream stream) throws IOException { + return readMagicHeader(stream, getMinLength()); + } + + /** + * Like {@link #readMagicHeader(InputStream)} but sized by {@code maxBytes} when the + * caller knows the stream is shorter than {@link #getMinLength()} -- avoids a fresh + * 64KB allocation per detection for every small (e.g. embedded) document. + */ + byte[] readMagicHeader(InputStream stream, int maxBytes) throws IOException { if (stream == null) { throw new IllegalArgumentException("InputStream is missing"); } + if (maxBytes == 0) { + return new byte[0]; + } - byte[] bytes = new byte[getMinLength()]; + byte[] bytes = new byte[maxBytes]; int totalRead = 0; int lastRead = stream.read(bytes); @@ -364,7 +378,9 @@ public final class MimeTypes implements Detector, Serializable { if (mime == null) { mime = new MimeType(type); add(mime); - types.put(type, mime); + // add() indexed the raw type; index the normalized key too, or an + // aliased parameterized name would miss (and re-lock) on every call + types.putIfAbsent(normalisedType, mime); } } } @@ -525,9 +541,17 @@ public final class MimeTypes implements Detector, Serializable { // Get type based on magic prefix if (tis != null) { + int toRead = getMinLength(); + // hasLength() is non-forcing; getLength() would only spool when unknown + if (tis.hasLength()) { + long known = tis.getLength() - tis.getPosition(); + if (known >= 0 && known < toRead) { + toRead = (int) known; + } + } tis.mark(getMinLength()); try { - byte[] prefix = readMagicHeader(tis); + byte[] prefix = readMagicHeader(tis, toRead); possibleTypes = getMimeType(prefix); } finally { tis.reset(); @@ -541,21 +565,26 @@ public final class MimeTypes implements Detector, Serializable { boolean isHttp = false; // Deal with a URI or a path name in as the resource name - try { - URI uri = new URI(resourceName); - String scheme = uri.getScheme(); - isHttp = scheme != null && scheme.startsWith("http"); // http or https - String path = uri.getPath(); - if (path != null) { - int slash = path.lastIndexOf('/'); - if (slash + 1 < path.length()) { - name = path.substring(slash + 1); + // A space guarantees URISyntaxException; skip the constructor (and its + // exception fill-in) for that common embedded-resource case. + if (resourceName.indexOf(' ') >= 0) { + name = resourceName; + } else { + try { + URI uri = new URI(resourceName); + String scheme = uri.getScheme(); + isHttp = scheme != null && scheme.startsWith("http"); // http or https + String path = uri.getPath(); + if (path != null) { + int slash = path.lastIndexOf('/'); + if (slash + 1 < path.length()) { + name = path.substring(slash + 1); + } } + } catch (URISyntaxException e) { + name = resourceName; } - } catch (URISyntaxException e) { - name = resourceName; } - if (name != null) { MimeType hint = getMimeType(name); diff --git a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java index 48c0329f06..8cc8ce9728 100644 --- a/tika-core/src/main/java/org/apache/tika/mime/Patterns.java +++ b/tika-core/src/main/java/org/apache/tika/mime/Patterns.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import java.util.regex.Pattern; /** * Defines a MimeType pattern. @@ -49,6 +50,13 @@ class Patterns implements Serializable { */ private final SortedMap<String, MimeType> globs = new TreeMap<>(new LengthComparator()); + + /** + * Compiled forms of {@link #globs}' keys. Matching recompiled every glob regex + * per lookup before; for names that miss the name/extension indexes that was + * a Pattern.compile per glob per call. + */ + private final Map<String, Pattern> compiledGlobs = new HashMap<>(); private int minExtensionLength = Integer.MAX_VALUE; private int maxExtensionLength = 0; @@ -116,6 +124,7 @@ class Patterns implements Serializable { MimeType previous = globs.get(glob); if (previous == null || registry.isSpecializationOf(previous.getType(), type.getType())) { globs.put(glob, type); + compiledGlobs.put(glob, Pattern.compile(glob)); } else if (previous == type || registry.isSpecializationOf(type.getType(), previous.getType())) { // do nothing @@ -159,7 +168,10 @@ class Patterns implements Serializable { // And finally, try complex regexp matching for (Map.Entry<String, MimeType> entry : globs.entrySet()) { - if (name.matches(entry.getKey())) { + Pattern glob = compiledGlobs.get(entry.getKey()); + boolean matched = glob != null ? glob.matcher(name).matches() + : name.matches(entry.getKey()); + if (matched) { return entry.getValue(); } } diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml index 62a18b7840..9cb4a899bc 100644 --- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml +++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml @@ -312,7 +312,11 @@ <!-- we can do more specific versions by focusing on the integer in the regex below --> <tika:link>http://justsolve.archiveteam.org/wiki/Adobe_Illustrator_Artwork</tika:link> <magic priority="60"> - <match value="[\r\n]%AI5_FileFormat [1-4][\r\n]" type="regex" offset="0:8192"/> + <!-- literal gate first: the ranged regex only runs when the marker exists, + so non-AI files pay a cheap string scan instead of 8K regex attempts --> + <match value="%AI5_FileFormat " type="string" offset="0:8192"> + <match value="[\r\n]%AI5_FileFormat [1-4][\r\n]" type="regex" offset="0:8192"/> + </match> </magic> <sub-class-of type="application/postscript"/> </mime-type>
