This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4782 in repository https://gitbox.apache.org/repos/asf/tika.git
commit f4fd89c898023094ec3f1ee6774b1ec15a3eabb5 Author: tallison <[email protected]> AuthorDate: Sun Aug 16 21:02:42 2026 -0400 TIKA_4782 -- allow print ticket headers in pdf detection --- CHANGES.txt | 5 + .../org/apache/tika/mime/tika-mimetypes.xml | 10 ++ .../org/apache/tika/mime/PdfDetectionTest.java | 172 +++++++++++++++++++++ .../mime/test-pdf-with-print-ticket-header.pdf | Bin 0 -> 36766 bytes 4 files changed, 187 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index d736d877e8..5ae4bb52ed 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -151,6 +151,11 @@ Release 4.0.0 - ??? closes its connection and recycles the per-client worker, so a pooled client cannot go back to the queue dirty (TIKA-4815). + * PDFs whose %PDF- header is preceded by a print-composition job ticket are + no longer detected as text/x-matlab. Up to 50 %% comment or blank lines of + up to 150 characters, and nothing else, may now precede the header; the + TIKA-3328 rule this extends only reached 512 bytes (TIKA-4782). + Release 4.0.0-beta-1 - 6/29/2026 diff --git a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml index 6cd798ca80..a2f4374f4f 100644 --- a/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml +++ b/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml @@ -855,6 +855,16 @@ <match value="%PDF-2." type="string" offset="1:512"/> </match> </magic> + <magic priority="40"> + <!-- Print-composition systems prepend a job ticket of %% comments that can run well + past the 512-byte window above (TIKA-4782). Up to 50 %% comment or blank lines of up + to 150 characters, and nothing else, may precede the header. 50x150 is deliberate: + the worst case stays inside the first 8K, which is all MagicDetector feeds a regex, + so every prefix within those bounds really is reachable. + Every quantifier is bounded and the group is atomic, so this cannot backtrack: without + the (?> a hostile CRLF run splits exponentially and hangs detection. Keep it that way. --> + <match value="(?>(?:%%[^\\r\\n]{0,148})?(?:\\r\\n|[\\r\\n])){1,50}%PDF-[12]\\." type="regex" offset="0"/> + </magic> <magic priority="20"> <!-- Low priority match for %PDF-#.# near the start of the file --> <!-- Can trigger false positives, so set the priority rather low here --> diff --git a/tika-core/src/test/java/org/apache/tika/mime/PdfDetectionTest.java b/tika-core/src/test/java/org/apache/tika/mime/PdfDetectionTest.java new file mode 100644 index 0000000000..3e5bd1f10b --- /dev/null +++ b/tika-core/src/test/java/org/apache/tika/mime/PdfDetectionTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.mime; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; + +import java.io.InputStream; +import java.time.Duration; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.apache.tika.io.TikaInputStream; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.ParseContext; + +/** + * Detection of PDFs whose {@code %PDF-} header is preceded by {@code %%} comment + * lines, which would otherwise be claimed by the matlab {@code %%} magic. + * + * @see <a href="https://issues.apache.org/jira/browse/TIKA-3328";>TIKA-3328</a> + * @see <a href="https://issues.apache.org/jira/browse/TIKA-4782";>TIKA-4782</a> + */ +public class PdfDetectionTest { + + private static final MediaType PDF = MediaType.application("pdf"); + + private static final MediaType MATLAB = MediaType.text("x-matlab"); + + private static final String PDF_BODY = "%PDF-1.7\r\n1 0 obj\r\n"; + + private static MimeTypes MIME_TYPES; + + @BeforeAll + public static void setUp() { + MIME_TYPES = MimeTypes.getDefaultMimeTypes(); + } + + /** + * Print-shop job ticket ahead of the header; the {@code %PDF-} lands well past + * the 512-byte window of the older TIKA-3328 rule. + */ + @Test + public void testPrintTicketHeader() throws Exception { + try (InputStream in = getClass().getResourceAsStream("test-pdf-with-print-ticket-header.pdf")) { + assertNotNull(in, "missing test file"); + assertEquals(PDF, detect(in)); + } + } + + /** + * 10 and 50 lines of 60 push the header past the 512-byte window the older rules + * can reach; 1 line keeps TIKA-3328 covered. + */ + @Test + public void testCommentLinesBeforeHeader() throws Exception { + for (int lines : new int[]{1, 10, 50}) { + assertEquals(PDF, detect(commentLines(lines, 60) + PDF_BODY), lines + " comment lines"); + } + assertEquals(PDF, detect(commentLines(5, 150) + PDF_BODY), "maximum-length lines"); + } + + /** + * Both bounds at once. This is the case that catches a prefix sized within the + * documented bounds but past the 8K MagicDetector hands a regex. + */ + @Test + public void testLargestAcceptedPrefix() throws Exception { + assertEquals(PDF, detect(commentLines(50, 150) + PDF_BODY)); + } + + @Test + public void testBlankLinesAndLineEndings() throws Exception { + assertEquals(PDF, detect("%%BeginTicket\r\n\r\n%%EndTicket\n\n" + PDF_BODY)); + assertEquals(PDF, detect("\r\n\r\n" + PDF_BODY)); + assertEquals(PDF, detect("%%a\r%%b\r" + PDF_BODY)); + assertEquals(PDF, detect("%%a\n%%b\n" + PDF_BODY.replace("%PDF-1.", "%PDF-2."))); + } + + /** + * Each negative case puts the header past 512 bytes, so only the TIKA-4782 rule + * could have matched it. + */ + @Test + public void testCommentPrefixIsBounded() throws Exception { + assertNotEquals(PDF, detect(commentLines(51, 60) + PDF_BODY), "51 comment lines"); + assertNotEquals(PDF, detect(commentLines(5, 151) + PDF_BODY), "over-long comment lines"); + } + + /** + * Only comment and blank lines may precede the header: anything else and this is + * some other format that happens to embed a PDF. + */ + @Test + public void testNonCommentPrefixIsNotPdf() throws Exception { + assertNotEquals(PDF, detect(commentLines(10, 60) + "x = 1;\r\n" + PDF_BODY)); + } + + @Test + public void testMatlabStillDetected() throws Exception { + assertEquals(MATLAB, detect("%% cell one\r\nx = 1;\r\n%% cell two\r\ny = x + 1;\r\n")); + } + + /** + * The TIKA-4782 regex must not backtrack. Earlier drafts of it hung Java's matcher + * indefinitely on these inputs; linear forms answer in well under a millisecond, so + * a generous timeout separates the two without being timing-sensitive. + */ + @Test + public void testNoCatastrophicBacktracking() { + String[] hostile = new String[]{ + "\r\n".repeat(4096), + "\r".repeat(8192), + "%".repeat(8192), + "%%".repeat(4096), + "%%a\r\n".repeat(1638), + "%%a\r\n\r\n".repeat(1024), + commentLines(50, 150) + "\r\n".repeat(1000) + }; + assertTimeoutPreemptively(Duration.ofSeconds(10), () -> { + for (String s : hostile) { + detect(s); + } + }); + } + + /** + * @param lineLength characters per line including the leading {@code %%}, excluding the CRLF + */ + private static String commentLines(int count, int lineLength) { + StringBuilder line = new StringBuilder("%%"); + while (line.length() < lineLength) { + line.append('A'); + } + line.append("\r\n"); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < count; i++) { + sb.append(line); + } + return sb.toString(); + } + + private static MediaType detect(String bytes) throws Exception { + try (TikaInputStream tis = TikaInputStream.get(bytes.getBytes(ISO_8859_1))) { + return MIME_TYPES.detect(tis, new Metadata(), new ParseContext()); + } + } + + private static MediaType detect(InputStream in) throws Exception { + try (TikaInputStream tis = TikaInputStream.get(in)) { + return MIME_TYPES.detect(tis, new Metadata(), new ParseContext()); + } + } +} diff --git a/tika-core/src/test/resources/org/apache/tika/mime/test-pdf-with-print-ticket-header.pdf b/tika-core/src/test/resources/org/apache/tika/mime/test-pdf-with-print-ticket-header.pdf new file mode 100644 index 0000000000..a23b0c6c4d Binary files /dev/null and b/tika-core/src/test/resources/org/apache/tika/mime/test-pdf-with-print-ticket-header.pdf differ
