This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 024ecefc1f TIKA-5011: keep LaTeX math verbatim in VLM Markdown output
(#3455)
024ecefc1f is described below
commit 024ecefc1f0bb9643664275f1c73f6d4020c1108
Author: Tim Allison <[email protected]>
AuthorDate: Fri Oct 30 11:43:43 2026 -0400
TIKA-5011: keep LaTeX math verbatim in VLM Markdown output (#3455)
---
CHANGES.txt | 8 +
.../pages/configuration/inference-recipes.adoc | 4 +-
.../apache/tika/sax/ToMarkdownContentHandler.java | 51 ++++
.../tika/sax/ToMarkdownContentHandlerTest.java | 45 ++++
.../tika/parser/vlm/MarkdownToXHTMLEmitter.java | 47 +++-
.../java/org/apache/tika/parser/vlm/MathSpans.java | 287 +++++++++++++++++++++
.../parser/vlm/MarkdownToXHTMLEmitterTest.java | 93 +++++++
7 files changed, 521 insertions(+), 14 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 2a8e7586d8..d010544521 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -44,6 +44,14 @@ Release 4.3.0 - ???
async emitter threads) shared one connection and prepared statement
unguarded,
mixing values across rows and committing each other's batches (TIKA-5007).
+ EXTRACTION AND FORMATS
+
+ * VLM text recognizers (jina-ocr, OpenAI-compatible, Claude, Gemini) no
longer corrupt
+ LaTeX math in model output: $...$, $$...$$, \(...\) and \[...\] are kept
verbatim
+ instead of being read as Markdown, which dropped the backslash from \{,
\} and \|
+ and split GFM table cells on |. Math is emitted as <span class="math
inline"> or
+ <span class="math display">, and the Markdown writer outputs it unescaped
(TIKA-5011).
+
OTHER CHANGES
* A runtime fetcher or emitter save or delete reaches running pipes workers
at
diff --git a/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
b/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
index a5045a51e7..2acebc7204 100644
--- a/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
+++ b/docs/modules/ROOT/pages/configuration/inference-recipes.adoc
@@ -65,7 +65,9 @@ text is poor. The key comes from the environment
The model returns markdown with tables as raw HTML, whatever the prompt says;
Tika parses both
into XHTML, so a table is a `<table>` in every output format and a pipe table
in markdown. Body
-cells come back as `<th>` and are passed through as such. The load-time health
check reads the
+cells come back as `<th>` and are passed through as such. Math comes back as
LaTeX; Tika keeps it
+verbatim, delimiters included, in `<span class="math inline">` or `<span
class="math display">`,
+and writes it unescaped in markdown. The load-time health check reads the
public `/v1/models`, so a wrong key passes startup and surfaces on the first
request as the
document's parse exception (HTTP 401 in the message).
diff --git
a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
index 8be96fd00a..42469f6cd2 100644
--- a/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
+++ b/tika-core/src/main/java/org/apache/tika/sax/ToMarkdownContentHandler.java
@@ -45,6 +45,7 @@ import org.commonmark.node.Emphasis;
import org.commonmark.node.FencedCodeBlock;
import org.commonmark.node.HardLineBreak;
import org.commonmark.node.Heading;
+import org.commonmark.node.HtmlInline;
import org.commonmark.node.Image;
import org.commonmark.node.Link;
import org.commonmark.node.ListBlock;
@@ -105,6 +106,10 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
private Code inlineCode;
private StringBuilder inlineCodeText;
+ // Buffer for <span class="math ..."> TeX, written verbatim; depth counts
nested spans.
+ private StringBuilder mathText;
+ private int mathSpanDepth;
+
// Buffer for <pre> fenced code block literal.
private FencedCodeBlock codeBlock;
private StringBuilder codeBlockText;
@@ -253,6 +258,15 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
ensureInlineContainer();
stack.peek().appendChild(new HardLineBreak());
break;
+ case "span":
+ if (mathText != null) {
+ mathSpanDepth++;
+ } else if (codeBlockText == null && inlineCodeText == null
+ && isMathClass(atts.getValue("class"))) {
+ mathText = new StringBuilder();
+ mathSpanDepth = 1;
+ }
+ break;
case "img":
ensureInlineContainer();
Image img = new Image(value(atts, "src"), null);
@@ -330,6 +344,12 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
case "div":
closeImplicitParagraph();
break;
+ case "span":
+ if (mathText != null && --mathSpanDepth == 0) {
+ appendMath(mathText.toString());
+ mathText = null;
+ }
+ break;
case "table":
endTable();
break;
@@ -363,6 +383,10 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
inlineCodeText.append(ch, start, length);
return;
}
+ if (mathText != null) {
+ mathText.append(ch, start, length);
+ return;
+ }
String text = new String(ch, start, length);
if (text.trim().isEmpty()) {
@@ -637,6 +661,33 @@ public class ToMarkdownContentHandler extends
DefaultHandler {
stack.push(cell);
}
+ /**
+ * TeX goes out raw: the renderer's escaping ({@code \\mathcal}, {@code
u\\_0}) breaks
+ * math-aware readers. In a table cell the renderer still escapes {@code
|}, which GFM
+ * unescapes on read.
+ */
+ private void appendMath(String tex) {
+ if (tex.isEmpty()) {
+ return;
+ }
+ ensureInlineContainer();
+ HtmlInline raw = new HtmlInline();
+ raw.setLiteral(collapseLineBreaks(tex));
+ stack.peek().appendChild(raw);
+ }
+
+ private static boolean isMathClass(String cls) {
+ if (cls == null) {
+ return false;
+ }
+ for (String token : cls.trim().split("\\s+")) {
+ if (token.equals("math")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private static BulletList newBulletList() {
BulletList list = new BulletList();
list.setMarker("-");
diff --git
a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
index 2b060e504d..90456834e4 100644
---
a/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
+++
b/tika-core/src/test/java/org/apache/tika/sax/ToMarkdownContentHandlerTest.java
@@ -500,6 +500,51 @@ public class ToMarkdownContentHandlerTest {
assertTrue(handler.toString().contains("Line one \nLine two"));
}
+ @Test
+ public void testMathSpanWrittenVerbatim() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ chars(handler, "see u_0 and ");
+ startElement(handler, "span", "class", "math inline");
+ chars(handler, "$u_0 \\in ");
+ startElement(handler, "span");
+ chars(handler, "\\mathcal{H} *");
+ endElement(handler, "span");
+ chars(handler, "\n x$");
+ endElement(handler, "span");
+ endElement(handler, "p");
+ handler.endDocument();
+ String md = handler.toString();
+ assertContains("see u\\_0 and $u_0 \\in \\mathcal{H} * x$", md);
+ }
+
+ @Test
+ public void testMathSpanInTableCellEscapesOnlyPipes() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ script(handler, "table", "tr", "th", "#h1", "/th", "th", "#h2", "/th",
"/tr", "tr", "td");
+ startElement(handler, "span", "class", "math inline");
+ chars(handler, "$\\|x\\|_1$");
+ endElement(handler, "span");
+ script(handler, "/td", "td", "#2", "/td", "/tr", "/table");
+ handler.endDocument();
+ assertContains("|$\\\\|x\\\\|_1$|2|", handler.toString());
+ }
+
+ @Test
+ public void testNonMathSpanStillEscaped() throws Exception {
+ ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
+ handler.startDocument();
+ startElement(handler, "p");
+ startElement(handler, "span", "class", "mathematics");
+ chars(handler, "u_0*");
+ endElement(handler, "span");
+ endElement(handler, "p");
+ handler.endDocument();
+ assertContains("u\\_0\\*", handler.toString());
+ }
+
@Test
public void testBoldInsideListItem() throws Exception {
ToMarkdownContentHandler handler = new ToMarkdownContentHandler();
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
index 9c1073ce65..b2a047d44d 100644
---
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitter.java
@@ -82,6 +82,9 @@ import org.xml.sax.helpers.AttributesImpl;
* attributes dropped except {@code colspan}/{@code rowspan}; unknown
elements
* contribute their text. Inline HTML tags are
* dropped (their text is kept), except {@code <br>}.</li>
+ * <li>TeX math ({@code $...$}, {@code $$...$$}, {@code \(...\)}, {@code
\[...\]}):
+ * kept verbatim, delimiters included, in {@code <span class="math
inline">} or
+ * {@code <span class="math display">} (see {@link MathSpans}).</li>
* </ul>
*
* @since Apache Tika 4.0
@@ -133,8 +136,9 @@ class MarkdownToXHTMLEmitter {
if (markdown == null || markdown.isEmpty()) {
return;
}
- Node document = PARSER.parse(markdown);
- SAXVisitor visitor = new SAXVisitor(handler);
+ MathSpans math = MathSpans.protect(markdown);
+ Node document = PARSER.parse(math.text());
+ SAXVisitor visitor = new SAXVisitor(handler, math);
document.accept(visitor);
if (visitor.saxException != null) {
throw visitor.saxException;
@@ -147,10 +151,12 @@ class MarkdownToXHTMLEmitter {
private static class SAXVisitor extends AbstractVisitor {
private final ContentHandler handler;
+ private final MathSpans math;
SAXException saxException;
- SAXVisitor(ContentHandler handler) {
+ SAXVisitor(ContentHandler handler, MathSpans math) {
this.handler = handler;
+ this.math = math;
}
// --- block nodes ---
@@ -221,7 +227,7 @@ class MarkdownToXHTMLEmitter {
}
startElement("pre");
startElement("code", attrs);
- characters(fencedCodeBlock.getLiteral());
+ characters(math.restore(fencedCodeBlock.getLiteral()));
endElement("code");
endElement("pre");
}
@@ -230,7 +236,7 @@ class MarkdownToXHTMLEmitter {
public void visit(IndentedCodeBlock indentedCodeBlock) {
startElement("pre");
startElement("code");
- characters(indentedCodeBlock.getLiteral());
+ characters(math.restore(indentedCodeBlock.getLiteral()));
endElement("code");
endElement("pre");
}
@@ -257,7 +263,7 @@ class MarkdownToXHTMLEmitter {
if (node instanceof TextNode) {
String text = ((TextNode) node).getWholeText();
if (!(inStructure && text.isBlank())) {
- characters(text);
+ textWithMath(text);
}
return;
}
@@ -303,7 +309,7 @@ class MarkdownToXHTMLEmitter {
@Override
public void visit(Text text) {
- characters(text.getLiteral());
+ textWithMath(text.getLiteral());
}
@Override
@@ -323,16 +329,16 @@ class MarkdownToXHTMLEmitter {
@Override
public void visit(Code code) {
startElement("code");
- characters(code.getLiteral());
+ characters(math.restore(code.getLiteral()));
endElement("code");
}
@Override
public void visit(Link link) {
AttributesImpl attrs = new AttributesImpl();
- attrs.addAttribute("", "href", "href", "CDATA",
link.getDestination());
+ attrs.addAttribute("", "href", "href", "CDATA",
math.restore(link.getDestination()));
if (link.getTitle() != null && !link.getTitle().isEmpty()) {
- attrs.addAttribute("", "title", "title", "CDATA",
link.getTitle());
+ attrs.addAttribute("", "title", "title", "CDATA",
math.restore(link.getTitle()));
}
startElement("a", attrs);
visitChildren(link);
@@ -342,9 +348,9 @@ class MarkdownToXHTMLEmitter {
@Override
public void visit(Image image) {
AttributesImpl attrs = new AttributesImpl();
- attrs.addAttribute("", "src", "src", "CDATA",
image.getDestination());
+ attrs.addAttribute("", "src", "src", "CDATA",
math.restore(image.getDestination()));
if (image.getTitle() != null && !image.getTitle().isEmpty()) {
- attrs.addAttribute("", "title", "title", "CDATA",
image.getTitle());
+ attrs.addAttribute("", "title", "title", "CDATA",
math.restore(image.getTitle()));
}
// Use alt text from child text nodes
StringBuilder alt = new StringBuilder();
@@ -355,7 +361,7 @@ class MarkdownToXHTMLEmitter {
}
child = child.getNext();
}
- attrs.addAttribute("", "alt", "alt", "CDATA", alt.toString());
+ attrs.addAttribute("", "alt", "alt", "CDATA",
math.restore(alt.toString()));
emptyElement("img", attrs);
}
@@ -443,6 +449,21 @@ class MarkdownToXHTMLEmitter {
}
}
+ private void textWithMath(String text) {
+ for (MathSpans.Piece piece : math.split(text)) {
+ if (!piece.math) {
+ characters(piece.text);
+ continue;
+ }
+ AttributesImpl attrs = new AttributesImpl();
+ attrs.addAttribute("", "class", "class", "CDATA",
+ piece.display ? "math display" : "math inline");
+ startElement("span", attrs);
+ characters(piece.text);
+ endElement("span");
+ }
+ }
+
// --- SAX helpers ---
private void startElement(String localName) {
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MathSpans.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MathSpans.java
new file mode 100644
index 0000000000..d73f5d33e1
--- /dev/null
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/main/java/org/apache/tika/parser/vlm/MathSpans.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.vlm;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Shields TeX math in model output from the Markdown parser.
+ * <p>
+ * CommonMark has no math syntax: inside {@code $\|x\|$} it reads {@code \|}
as an
+ * escaped pipe and drops the backslash, and a GFM table splits cells on the
pipes.
+ * {@link #protect} swaps each math span for a placeholder before parsing;
+ * {@link #restore} and {@link #split} put the original text back afterwards.
+ * <p>
+ * Delimiters: {@code $...$}, {@code $$...$$}, {@code \(...\)}, {@code
\[...\]}.
+ * {@code $...$} follows pandoc's guards so prices are not math: the opening
+ * {@code $} must be followed by a non-space, the closing one preceded by a
non-space
+ * and not followed by a digit. No span crosses a blank line or a backtick,
and code
+ * spans and fenced code blocks are left alone.
+ */
+final class MathSpans {
+
+ static final char OPEN = '';
+ static final char CLOSE = '';
+
+ /** One piece of text after {@link #split}: plain text, or a math span
with delimiters. */
+ static final class Piece {
+ final String text;
+ final boolean math;
+ final boolean display;
+
+ Piece(String text, boolean math, boolean display) {
+ this.text = text;
+ this.math = math;
+ this.display = display;
+ }
+ }
+
+ private final String text;
+ private final List<String> spans;
+
+ private MathSpans(String text, List<String> spans) {
+ this.text = text;
+ this.spans = spans;
+ }
+
+ /** The input with each math span replaced by a placeholder. */
+ String text() {
+ return text;
+ }
+
+ static MathSpans protect(String s) {
+ List<String> spans = new ArrayList<>();
+ if (s.indexOf(OPEN) >= 0 || s.indexOf(CLOSE) >= 0
+ || (s.indexOf('$') < 0 && s.indexOf('\\') < 0)) {
+ return new MathSpans(s, spans);
+ }
+ StringBuilder out = new StringBuilder(s.length());
+ // a failed search for a closer stops at a boundary; openers before it
fail too
+ int[] failedUntil = new int[4];
+ int n = s.length();
+ int i = 0;
+ while (i < n) {
+ if (i == 0 || s.charAt(i - 1) == '\n') {
+ int fenceEnd = skipFencedBlock(s, i);
+ if (fenceEnd > i) {
+ out.append(s, i, fenceEnd);
+ i = fenceEnd;
+ continue;
+ }
+ }
+ char c = s.charAt(i);
+ int end = -1;
+ if (c == '`') {
+ int skip = skipCodeSpan(s, i);
+ out.append(s, i, skip);
+ i = skip;
+ continue;
+ } else if (c == '\\' && i + 1 < n) {
+ char next = s.charAt(i + 1);
+ if (next == '(' || next == '[') {
+ int kind = next == '(' ? 0 : 1;
+ if (i >= failedUntil[kind]) {
+ end = findCloser(s, i + 2, next == '(' ? ')' : ']',
kind, failedUntil);
+ }
+ }
+ if (end < 0) {
+ out.append(s, i, i + 2);
+ i += 2;
+ continue;
+ }
+ } else if (c == '$') {
+ if (i + 1 < n && s.charAt(i + 1) == '$') {
+ if (i >= failedUntil[2]) {
+ end = findCloser(s, i + 2, '$', 2, failedUntil);
+ }
+ if (end < 0) {
+ out.append("$$");
+ i += 2;
+ continue;
+ }
+ } else if (i + 1 < n && !Character.isWhitespace(s.charAt(i +
1))
+ && i >= failedUntil[3]) {
+ end = findCloser(s, i + 1, '$', 3, failedUntil);
+ }
+ }
+ if (end < 0) {
+ out.append(c);
+ i++;
+ continue;
+ }
+ out.append(OPEN).append(spans.size()).append(CLOSE);
+ spans.add(s.substring(i, end));
+ i = end;
+ }
+ return new MathSpans(out.toString(), spans);
+ }
+
+ /**
+ * Finds the end (exclusive) of a math span whose content starts at {@code
from}.
+ * kind: 0 {@code \)}, 1 {@code \]}, 2 {@code $$}, 3 {@code $}. Returns
-1, and
+ * records the boundary in {@code failedUntil}, when there is none.
+ */
+ private static int findCloser(String s, int from, char close, int kind,
int[] failedUntil) {
+ int n = s.length();
+ int i = from;
+ while (i < n) {
+ char c = s.charAt(i);
+ if (c == '`' || (c == '\n' && isBlankLineAhead(s, i + 1))) {
+ break;
+ }
+ if (c == '\\') {
+ if (kind <= 1 && i + 1 < n && s.charAt(i + 1) == close) {
+ return i > from ? i + 2 : -1;
+ }
+ i += 2;
+ continue;
+ }
+ if (c == '$') {
+ if (kind == 2) {
+ if (i + 1 < n && s.charAt(i + 1) == '$' && i > from) {
+ return i + 2;
+ }
+ } else if (kind == 3) {
+ boolean afterNonSpace = !Character.isWhitespace(s.charAt(i
- 1));
+ boolean digitNext = i + 1 < n &&
Character.isDigit(s.charAt(i + 1));
+ if (afterNonSpace && !digitNext) {
+ return i + 1;
+ }
+ }
+ }
+ i++;
+ }
+ failedUntil[kind] = i;
+ return -1;
+ }
+
+ private static boolean isBlankLineAhead(String s, int i) {
+ while (i < s.length() && (s.charAt(i) == ' ' || s.charAt(i) == '\t')) {
+ i++;
+ }
+ return i >= s.length() || s.charAt(i) == '\n';
+ }
+
+ /** End of a backtick code span starting at i, or the end of the backtick
run. */
+ private static int skipCodeSpan(String s, int i) {
+ int run = runLength(s, i, '`');
+ int j = i + run;
+ while (j < s.length()) {
+ int k = s.indexOf('`', j);
+ if (k < 0) {
+ break;
+ }
+ int len = runLength(s, k, '`');
+ if (len == run) {
+ return k + len;
+ }
+ j = k + len;
+ }
+ return i + run;
+ }
+
+ /** End of a fenced code block opening at line start i, or i if there is
none. */
+ private static int skipFencedBlock(String s, int i) {
+ int j = i;
+ while (j < s.length() && j - i < 3 && s.charAt(j) == ' ') {
+ j++;
+ }
+ if (j >= s.length() || (s.charAt(j) != '`' && s.charAt(j) != '~')) {
+ return i;
+ }
+ char fence = s.charAt(j);
+ int len = runLength(s, j, fence);
+ if (len < 3) {
+ return i;
+ }
+ int line = s.indexOf('\n', j);
+ while (line >= 0) {
+ int k = line + 1;
+ while (k < s.length() && k - line - 1 < 3 && s.charAt(k) == ' ') {
+ k++;
+ }
+ if (runLength(s, k, fence) >= len) {
+ int eol = s.indexOf('\n', k);
+ return eol < 0 ? s.length() : eol + 1;
+ }
+ line = s.indexOf('\n', k);
+ }
+ return s.length();
+ }
+
+ private static int runLength(String s, int i, char c) {
+ int j = i;
+ while (j < s.length() && s.charAt(j) == c) {
+ j++;
+ }
+ return j - i;
+ }
+
+ /** Replaces placeholders in a literal with the original math text. */
+ String restore(String s) {
+ if (s == null || spans.isEmpty() || s.indexOf(OPEN) < 0) {
+ return s;
+ }
+ StringBuilder sb = new StringBuilder(s.length());
+ for (Piece p : split(s)) {
+ sb.append(p.text);
+ }
+ return sb.toString();
+ }
+
+ /** Splits text at placeholders into plain and math pieces. */
+ List<Piece> split(String s) {
+ List<Piece> pieces = new ArrayList<>();
+ int start = 0;
+ int i = spans.isEmpty() ? -1 : s.indexOf(OPEN);
+ while (i >= 0) {
+ int close = s.indexOf(CLOSE, i);
+ int index = close < 0 ? -1 : parseIndex(s, i + 1, close);
+ if (index < 0) {
+ i = s.indexOf(OPEN, i + 1);
+ continue;
+ }
+ if (i > start) {
+ pieces.add(new Piece(s.substring(start, i), false, false));
+ }
+ String math = spans.get(index);
+ pieces.add(new Piece(math, true, math.startsWith("$$") ||
math.startsWith("\\[")));
+ start = close + 1;
+ i = s.indexOf(OPEN, start);
+ }
+ if (start < s.length()) {
+ pieces.add(new Piece(s.substring(start), false, false));
+ }
+ return pieces;
+ }
+
+ private int parseIndex(String s, int from, int to) {
+ if (to <= from || to - from > 9) {
+ return -1;
+ }
+ int v = 0;
+ for (int k = from; k < to; k++) {
+ char c = s.charAt(k);
+ if (c < '0' || c > '9') {
+ return -1;
+ }
+ v = v * 10 + (c - '0');
+ }
+ return v < spans.size() ? v : -1;
+ }
+}
diff --git
a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
index 783b4f020d..6e4b549bf4 100644
---
a/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
+++
b/tika-parsers/tika-parsers-ml/tika-vlm/src/test/java/org/apache/tika/parser/vlm/MarkdownToXHTMLEmitterTest.java
@@ -18,9 +18,11 @@ package org.apache.tika.parser.vlm;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.StringWriter;
+import java.time.Duration;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
@@ -29,6 +31,8 @@ import javax.xml.transform.stream.StreamResult;
import org.apache.commons.xml.secure.SecureTransformerFactory;
import org.junit.jupiter.api.Test;
+import org.apache.tika.sax.ToMarkdownContentHandler;
+
public class MarkdownToXHTMLEmitterTest {
@Test
@@ -292,6 +296,95 @@ public class MarkdownToXHTMLEmitterTest {
assertContains("<p>This is bold, x2 and a<br/>break.</p>", xml);
}
+ @Test
+ void testInlineMathKeepsBackslashedPunctuation() throws Exception {
+ String xml = emit("the norm $\\|\\cdot\\|$ on $S := \\{u \\in
\\mathcal{H}\\}$ here");
+ assertContains("<span class=\"math inline\">$\\|\\cdot\\|$</span>",
xml);
+ assertContains("<span class=\"math inline\">$S := \\{u \\in
\\mathcal{H}\\}$</span>", xml);
+ }
+
+ @Test
+ void testMathDelimiters() throws Exception {
+ assertContains("<span class=\"math inline\">\\(\\{x\\}\\)</span>",
emit("a \\(\\{x\\}\\) b"));
+ assertContains("<span class=\"math display\">\\[\\|x\\|\\]</span>",
emit("\\[\\|x\\|\\]"));
+ assertContains("<span class=\"math display\">$$a_1 * b_2$$</span>",
emit("$$a_1 * b_2$$"));
+ }
+
+ @Test
+ void testMultiLineDisplayMathIsNotParsedAsBlocks() throws Exception {
+ String xml = emit("$$\nx = a\n- b\n# c\n$$\n\nafter");
+ assertContains("<span class=\"math display\">$$\nx = a\n- b\n#
c\n$$</span>", xml);
+ assertFalse(xml.contains("<li>"), xml);
+ assertFalse(xml.contains("<h1>"), xml);
+ }
+
+ @Test
+ void testMathKeepsEmphasisMarkersLiteral() throws Exception {
+ String xml = emit("$a_1 * b_2 * c_3$ and *em*");
+ assertContains("<span class=\"math inline\">$a_1 * b_2 * c_3$</span>",
xml);
+ assertContains("<i>em</i>", xml);
+ }
+
+ @Test
+ void testDollarsThatAreNotMath() throws Exception {
+ for (String md : new String[] {"costs $5 and $10 today", "a $ b$ c",
"a $b $ c",
+ "pay $x$5", "lone $$ here", "escaped \\$x\\$ here", "split
$a\n\nb$"}) {
+ String xml = emit(md);
+ assertFalse(xml.contains("math"), md + " -> " + xml);
+ }
+ assertContains("costs $5 and $10 today", emit("costs $5 and $10
today"));
+ }
+
+ @Test
+ void testMathInsideCodeIsUntouched() throws Exception {
+ assertContains("<code>$\\|x\\|$</code>", emit("see `$\\|x\\|$` here"));
+ assertContains("<code>$\\{a\\}$\n</code>",
emit("```\n$\\{a\\}$\n```"));
+ // a code span that opens first wins over a later $
+ String xml = emit("`cost $5` and $x$");
+ assertContains("<code>cost $5</code>", xml);
+ assertContains("<span class=\"math inline\">$x$</span>", xml);
+ }
+
+ @Test
+ void testMathWithPipesInGfmTable() throws Exception {
+ String xml = emit("| a | b |\n|---|---|\n| $\\|x\\|$ | 2 |\n");
+ assertContains("<td><span class=\"math
inline\">$\\|x\\|$</span></td><td>2</td>", xml);
+ }
+
+ @Test
+ void testMathInHtmlTableBlock() throws Exception {
+ String xml = emit("<table><tr><td>$\\{a\\} < b$</td></tr></table>");
+ assertContains("<td><span class=\"math inline\">$\\{a\\} <
b$</span></td>", xml);
+ }
+
+ @Test
+ void testPlaceholderCharactersInInputPassThrough() throws Exception {
+ String md = "odd \uE000" + "0\uE001 and $\\{x\\}$";
+ String xml = emit(md);
+ assertContains("odd \uE000" + "0\uE001 and $", xml);
+ assertFalse(xml.contains("<span"), xml);
+ }
+
+ @Test
+ void testManyUnmatchedDollarsIsLinear() {
+ String md = "$a ".repeat(200_000);
+ assertTimeoutPreemptively(Duration.ofSeconds(20), () -> emit(md));
+ }
+
+ @Test
+ void testMathRoundTripsThroughMarkdownWriter() throws Exception {
+ String md = "norm $\\|\\cdot\\|$ with $u_0 \\in \\mathcal{H}$\n\n"
+ + "| a | b |\n|---|---|\n| $\\|x\\|$ | 2 |\n";
+ ToMarkdownContentHandler out = new ToMarkdownContentHandler();
+ out.startDocument();
+ MarkdownToXHTMLEmitter.emit(md, out);
+ out.endDocument();
+ String written = out.toString();
+ assertContains("norm $\\|\\cdot\\|$ with $u_0 \\in \\mathcal{H}$",
written);
+ // GFM unescapes \| in a cell before math, so a renderer sees $\|x\|$
+ assertContains("|$\\\\|x\\\\|$|2|", written);
+ }
+
private static final String XHTML_NS = "http://www.w3.org/1999/xhtml";;
/**
