This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new c66a83c2b3 TIKA-5016: keep superscript/subscript as sup/sub in docx,
pptx, doc, RTF and HTML (#3467)
c66a83c2b3 is described below
commit c66a83c2b3be6a7595758ab3aae85e07ea23d248
Author: Tim Allison <[email protected]>
AuthorDate: Fri Oct 30 22:23:22 2026 -0400
TIKA-5016: keep superscript/subscript as sup/sub in docx, pptx, doc, RTF
and HTML (#3467)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
CHANGES.txt | 5 +
.../apache/tika/parser/html/DefaultHtmlMapper.java | 4 +
.../apache/tika/parser/html/HtmlParserTest.java | 10 ++
.../tika/parser/microsoft/FormattingUtils.java | 8 +-
.../microsoft/ooxml/FormattingTagManager.java | 37 ++++-
.../microsoft/ooxml/OOXMLTikaBodyPartHandler.java | 6 +
.../ooxml/OOXMLWordAndPowerPointTextHandler.java | 11 ++
.../tika/parser/microsoft/ooxml/RunProperties.java | 39 ++++++
.../microsoft/ooxml/xwpf/XWPFStylesShim.java | 23 ++++
.../tika/parser/microsoft/rtf/GroupState.java | 6 +
.../tika/parser/microsoft/rtf/TextExtractor.java | 71 +++++++++-
.../tika/parser/microsoft/WordParserTest.java | 3 +-
.../parser/microsoft/ooxml/OOXMLDocxSAXTest.java | 4 +-
.../parser/microsoft/ooxml/OOXMLPptxSAXTest.java | 4 +-
.../parser/microsoft/ooxml/OOXMLSupSubTest.java | 153 +++++++++++++++++++++
.../tika/parser/microsoft/rtf/RTFParserTest.java | 34 +++++
16 files changed, 407 insertions(+), 11 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 4b6e4ebaf6..0ed5b18d8c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -50,6 +50,11 @@ Release 4.3.0 - ???
EXTRACTION AND FORMATS
+ * Superscript and subscript are kept as <sup>/<sub> in XHTML for docx
(w:vertAlign,
+ directly or through a character style), pptx (a:rPr baseline), doc, RTF
(\super,
+ \sub, \nosupersub) and HTML; they were dropped, so "x<sup>2</sup>" read
"x2" and
+ footnote markers fused with the word before them. .ppt is not covered
(TIKA-5016).
+
* VLM text recognizers (jina-ocr, OpenAI-compatible, Claude, Gemini) no
longer corrupt
LaTeX math in model output: $...$, $$...$$, \(...\) and \[...\] are kept
verbatim
instead of being read as Markdown, which dropped the backslash from \{,
\} and \|
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java
index 0056106e87..e9c6c03264 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/main/java/org/apache/tika/parser/html/DefaultHtmlMapper.java
@@ -79,6 +79,10 @@ public class DefaultHtmlMapper implements HtmlMapper {
put("PARAM", "param");
put("INS", "ins");
put("DEL", "del");
+
+ // not presentation: dropping them turns x<sup>2</sup> into "x2"
+ put("SUP", "sup");
+ put("SUB", "sub");
}};
private static final Set<String> DISCARDABLE_ELEMENTS = new
HashSet<String>() {{
add("STYLE");
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
index 20dd94097f..e39a8593d0 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-html-module/src/test/java/org/apache/tika/parser/html/HtmlParserTest.java
@@ -1431,4 +1431,14 @@ public class HtmlParserTest extends TikaTest {
assertContains("this is content", xml);
}
+
+ @Test
+ public void testSupSubKept() throws Exception {
+ String html = "<html><body><p>E = mc<SUP>2</SUP> and
H<sub>2</sub>O</p></body></html>";
+ String xml;
+ try (TikaInputStream tis = TikaInputStream.get(html.getBytes(UTF_8))) {
+ xml = getXML(tis, TikaTest.AUTO_DETECT_PARSER, new Metadata()).xml;
+ }
+ assertContains("E = mc<sup>2</sup> and H<sub>2</sub>O", xml);
+ }
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/FormattingUtils.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/FormattingUtils.java
index 8acb24a5bf..56c79253f1 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/FormattingUtils.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/FormattingUtils.java
@@ -89,6 +89,12 @@ public class FormattingUtils {
if (hwpfRun.getUnderlineCode() != 0) {
tags.add(Tag.U);
}
+ // 1 superscript, 2 subscript
+ if (hwpfRun.getSubSuperScriptIndex() == 1) {
+ tags.add(Tag.SUP);
+ } else if (hwpfRun.getSubSuperScriptIndex() == 2) {
+ tags.add(Tag.SUB);
+ }
}
return tags;
}
@@ -96,7 +102,7 @@ public class FormattingUtils {
public enum Tag {
// DON'T reorder elements to avoid breaking tests: EnumSet is iterated
in natural order
// as enum variants are declared
- B, I, S, U;
+ B, I, S, U, SUP, SUB;
public String tagName() {
return name().toLowerCase(Locale.ROOT);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java
index baf80e24aa..abab92c4e9 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/FormattingTagManager.java
@@ -19,6 +19,7 @@ package org.apache.tika.parser.microsoft.ooxml;
import java.util.Objects;
import org.apache.poi.xwpf.usermodel.UnderlinePatterns;
+import org.apache.poi.xwpf.usermodel.VerticalAlign;
import org.xml.sax.SAXException;
import org.apache.tika.sax.XHTMLContentHandler;
@@ -26,7 +27,8 @@ import org.apache.tika.sax.XHTMLContentHandler;
/**
* Single owner of all run-scoped XHTML wrapper tags, ensuring proper nesting.
* Nesting order from outermost to innermost:
- * {@code <a href="..."><b><i><s><u>text</u></s></i></b></a>}.
+ * {@code <a href="..."><b><i><s><u><sup>text</sup></u></s></i></b></a>}
+ * ({@code <sub>} in place of {@code <sup>}).
* <p>
* Hyperlinks come from two OOXML sources with different lifecycles:
* <ul>
@@ -55,6 +57,8 @@ class FormattingTagManager {
private boolean isItalics = false;
private boolean isStrikeThrough = false;
private boolean isUnderline = false;
+ // "sup", "sub" or null; innermost
+ private String vertTag = null;
FormattingTagManager(XHTMLContentHandler xhtml) {
this.xhtml = xhtml;
@@ -139,6 +143,16 @@ class FormattingTagManager {
currentSpanClass = spanClass;
}
+ boolean runIsUnderlined = runProperties.getUnderline() !=
UnderlinePatterns.NONE;
+ String runVertTag = toVertTag(runProperties.getVerticalAlign());
+ if (vertTag != null && (!vertTag.equals(runVertTag) ||
runProperties.isBold() != isBold
+ || runProperties.isItalics() != isItalics
+ || runProperties.isStrikeThrough() != isStrikeThrough
+ || runIsUnderlined != isUnderline)) {
+ xhtml.endElement(vertTag);
+ vertTag = null;
+ }
+
if (runProperties.isBold() != isBold) {
// Close inner tags before flipping <b>. Nesting is <b><i><s><u>
// (outermost to innermost), so close innermost first: u, s, i.
@@ -193,7 +207,6 @@ class FormattingTagManager {
isStrikeThrough = runProperties.isStrikeThrough();
}
- boolean runIsUnderlined = runProperties.getUnderline() !=
UnderlinePatterns.NONE;
if (runIsUnderlined != isUnderline) {
if (runIsUnderlined) {
xhtml.startElement("u");
@@ -202,6 +215,22 @@ class FormattingTagManager {
}
isUnderline = runIsUnderlined;
}
+
+ if (runVertTag != null && vertTag == null) {
+ xhtml.startElement(runVertTag);
+ vertTag = runVertTag;
+ }
+ }
+
+ private static String toVertTag(VerticalAlign align) {
+ switch (align) {
+ case SUPERSCRIPT:
+ return "sup";
+ case SUBSCRIPT:
+ return "sub";
+ default:
+ return null;
+ }
}
/**
@@ -252,6 +281,10 @@ class FormattingTagManager {
}
private void closeFormattingTags() throws SAXException {
+ if (vertTag != null) {
+ xhtml.endElement(vertTag);
+ vertTag = null;
+ }
if (isUnderline) {
xhtml.endElement("u");
isUnderline = false;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLTikaBodyPartHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLTikaBodyPartHandler.java
index f849f92d70..1c612e6b52 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLTikaBodyPartHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLTikaBodyPartHandler.java
@@ -146,6 +146,12 @@ public class OOXMLTikaBodyPartHandler
if (!runProperties.isItalicsSet()) {
effectiveRunProperties.italics = styles.isItalics(styleId);
}
+ if (!runProperties.isVerticalAlignSet()) {
+ String styleVertAlign = styles.getVerticalAlign(styleId);
+ if (styleVertAlign != null) {
+ effectiveRunProperties.setVerticalAlign(styleVertAlign);
+ }
+ }
formattingTags.applyFormatting(effectiveRunProperties,
getSpanClass(styleId));
}
xhtml.characters(contents);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
index 569b4065ae..1a0a20bd15 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/OOXMLWordAndPowerPointTextHandler.java
@@ -69,6 +69,7 @@ public class OOXMLWordAndPowerPointTextHandler extends
DefaultHandler {
private final static String I = "i";
private final static String U = "u";
private final static String STRIKE = "strike";
+ private final static String VERT_ALIGN = "vertAlign";
private final static String NUM_PR = "numPr";
private final static String BR = "br";
private final static String NO_BREAK_HYPHEN = "noBreakHyphen";
@@ -277,6 +278,12 @@ public class OOXMLWordAndPowerPointTextHandler extends
DefaultHandler {
//yes, I know, likely premature optimization...
if (RPR.equals(localName)) {
inRPr = true;
+ if (inR && DRAWING_MAIN_NS.equals(uri)) {
+ String baseline = atts.getValue("baseline");
+ if (baseline != null) {
+ currRunProperties.setBaselineOffset(baseline);
+ }
+ }
} else if (O_MATH.equals(localName) && M_NS.equals(uri)) {
if (mathDepth++ == 0) {
bodyContentsHandler.startMath();
@@ -321,6 +328,10 @@ public class OOXMLWordAndPowerPointTextHandler extends
DefaultHandler {
if (inR && inRPr) {
currRunProperties.setUnderline(getStringVal(atts));
}
+ } else if (VERT_ALIGN.equals(localName)) {
+ if (inR && inRPr) {
+ currRunProperties.setVerticalAlign(getStringVal(atts));
+ }
} else if (TR.equals(localName)) {
bodyContentsHandler.startTableRow();
} else if (NUM_PR.equals(localName)) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/RunProperties.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/RunProperties.java
index 56fbc6176e..e07c48d011 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/RunProperties.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/RunProperties.java
@@ -17,6 +17,7 @@
package org.apache.tika.parser.microsoft.ooxml;
import org.apache.poi.xwpf.usermodel.UnderlinePatterns;
+import org.apache.poi.xwpf.usermodel.VerticalAlign;
/**
* WARNING: This class is mutable. Make a copy of it
@@ -30,6 +31,9 @@ public class RunProperties {
UnderlinePatterns underline = UnderlinePatterns.NONE;
+ VerticalAlign verticalAlign = VerticalAlign.BASELINE;
+ boolean verticalAlignSet = false;
+
// PPTX hlinkClick hyperlink URL — set from <a:hlinkClick> inside <a:rPr>
String hlinkClickUrl = null;
@@ -95,6 +99,37 @@ public class RunProperties {
}
}
+ public VerticalAlign getVerticalAlign() {
+ return verticalAlign;
+ }
+
+ /** docx w:vertAlign w:val: superscript, subscript, baseline. */
+ public void setVerticalAlign(String val) {
+ verticalAlignSet = true;
+ if ("superscript".equals(val)) {
+ verticalAlign = VerticalAlign.SUPERSCRIPT;
+ } else if ("subscript".equals(val)) {
+ verticalAlign = VerticalAlign.SUBSCRIPT;
+ } else {
+ verticalAlign = VerticalAlign.BASELINE;
+ }
+ }
+
+ public boolean isVerticalAlignSet() {
+ return verticalAlignSet;
+ }
+
+ /** pptx a:rPr baseline, in thousandths of a percent: above 0 raised,
below 0 lowered. */
+ public void setBaselineOffset(String baseline) {
+ try {
+ int v = Integer.parseInt(baseline.trim());
+ verticalAlign = v > 0 ? VerticalAlign.SUPERSCRIPT
+ : v < 0 ? VerticalAlign.SUBSCRIPT : VerticalAlign.BASELINE;
+ } catch (NumberFormatException e) {
+ verticalAlign = VerticalAlign.BASELINE;
+ }
+ }
+
public String getHlinkClickUrl() {
return hlinkClickUrl;
}
@@ -108,6 +143,8 @@ public class RunProperties {
bold = other.bold;
strikeThrough = other.strikeThrough;
underline = other.underline;
+ verticalAlign = other.verticalAlign;
+ verticalAlignSet = other.verticalAlignSet;
hlinkClickUrl = other.hlinkClickUrl;
styleID = other.styleID;
boldSet = other.boldSet;
@@ -119,6 +156,8 @@ public class RunProperties {
bold = false;
strikeThrough = false;
underline = UnderlinePatterns.NONE;
+ verticalAlign = VerticalAlign.BASELINE;
+ verticalAlignSet = false;
hlinkClickUrl = null;
styleID = null;
boldSet = false;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
index c8f77dff47..78d541cb56 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/ooxml/xwpf/XWPFStylesShim.java
@@ -109,11 +109,27 @@ public class XWPFStylesShim {
return false;
}
+ /**
+ * @return the w:vertAlign value of the style or the nearest style it is
based on
+ * that sets one; null if none does
+ */
+ public String getVerticalAlign(String styleId) {
+ StyleInfo info = styles.get(styleId);
+ for (int i = 0; info != null && i < MAX_BASED_ON_DEPTH; i++) {
+ if (info.vertAlign != null) {
+ return info.vertAlign;
+ }
+ info = info.basedOn == null ? null : styles.get(info.basedOn);
+ }
+ return null;
+ }
+
private static class StyleInfo {
String name;
String basedOn;
Boolean bold;
Boolean italics;
+ String vertAlign;
}
private static class EmptyXWPFStyles extends XWPFStylesShim {
@@ -132,6 +148,11 @@ public class XWPFStylesShim {
public boolean isItalics(String styleId) {
return false;
}
+
+ @Override
+ public String getVerticalAlign(String styleId) {
+ return null;
+ }
}
private class StylesStripper extends DefaultHandler {
@@ -172,6 +193,8 @@ public class XWPFStylesShim {
current.bold = getOnOff(atts);
} else if (depth == 2 && inStyleRPr && "i".equals(localName)) {
current.italics = getOnOff(atts);
+ } else if (depth == 2 && inStyleRPr &&
"vertAlign".equals(localName)) {
+ current.vertAlign = getVal(atts);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/GroupState.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/GroupState.java
index 40ff44c94b..124bfa1479 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/GroupState.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/GroupState.java
@@ -22,9 +22,14 @@ import java.nio.charset.Charset;
* ... }. */
class GroupState {
+ static final int SUPER = 1;
+ static final int SUB = 2;
+
public int depth;
public boolean bold;
public boolean italic;
+ // 0 none, SUPER or SUB; nested inside <b><i>
+ public int vert;
// True if we are skipping all text in current group,
// eg if group leads with a \*:
public boolean ignore;
@@ -55,6 +60,7 @@ class GroupState {
public GroupState(GroupState other) {
bold = other.bold;
italic = other.italic;
+ vert = other.vert;
ignore = other.ignore;
ucSkip = other.ucSkip;
list = other.list;
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
index 1288c1ff51..2f6f4e26b2 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/main/java/org/apache/tika/parser/microsoft/rtf/TextExtractor.java
@@ -666,6 +666,12 @@ final class TextExtractor {
fieldState = 0;
hyperlinkAnchorDepth = -1;
}
+ if (groupState.vert != 0) {
+ end(vertTag(groupState.vert));
+ if (!preserveStyles) {
+ groupState.vert = 0;
+ }
+ }
if (groupState.italic) {
end("i");
groupState.italic = preserveStyles;
@@ -705,7 +711,7 @@ final class TextExtractor {
end(paragraphStack.pop());
}
}
- if (preserveStyles && (groupState.bold || groupState.italic)) {
+ if (preserveStyles && (groupState.bold || groupState.italic ||
groupState.vert != 0)) {
start(P);
pushParagraphTag(P);
startStyles(groupState);
@@ -973,6 +979,7 @@ final class TextExtractor {
if (fieldState == 0) {
if (groupState.bold) {
pushText();
+ endVert();
if (groupState.italic) {
end("i");
}
@@ -980,6 +987,7 @@ final class TextExtractor {
if (groupState.italic) {
start("i");
}
+ startVert();
groupState.bold = false;
}
}
@@ -990,8 +998,10 @@ final class TextExtractor {
if (fieldState == 0) {
if (groupState.italic) {
pushText();
+ endVert();
end("i");
groupState.italic = false;
+ startVert();
}
}
} else if (equals("f")) {
@@ -1212,8 +1222,9 @@ final class TextExtractor {
if (!groupState.bold) {
pushText();
lazyStartParagraph();
+ endVert();
if (groupState.italic) {
- // Make sure nesting is always <b><i>
+ // Make sure nesting is always <b><i><sup|sub>
end("i");
}
groupState.bold = true;
@@ -1224,8 +1235,25 @@ final class TextExtractor {
if (!groupState.italic) {
pushText();
lazyStartParagraph();
+ endVert();
groupState.italic = true;
start("i");
+ startVert();
+ }
+ } else if (equals("super") || equals("sub")) {
+ int vert = equals("super") ? GroupState.SUPER :
GroupState.SUB;
+ if (groupState.vert != vert) {
+ pushText();
+ lazyStartParagraph();
+ endVert();
+ groupState.vert = vert;
+ startVert();
+ }
+ } else if (equals("nosupersub")) {
+ if (groupState.vert != 0) {
+ pushText();
+ endVert();
+ groupState.vert = 0;
}
}
}
@@ -1240,7 +1268,7 @@ final class TextExtractor {
pendingListEnd();
}
} else if (equals("plain")) {
- if (groupState.italic || groupState.bold) {
+ if (groupState.italic || groupState.bold || groupState.vert != 0) {
// Reset styles
pushText();
endStyles(groupState);
@@ -1425,6 +1453,22 @@ final class TextExtractor {
}
}
+ private static String vertTag(int vert) {
+ return vert == GroupState.SUPER ? "sup" : "sub";
+ }
+
+ private void endVert() throws IOException, SAXException, TikaException {
+ if (groupState.vert != 0) {
+ end(vertTag(groupState.vert));
+ }
+ }
+
+ private void startVert() throws IOException, SAXException, TikaException {
+ if (groupState.vert != 0) {
+ start(vertTag(groupState.vert));
+ }
+ }
+
private void startStyles(GroupState groupState)
throws TikaException, IOException, SAXException {
//don't change styles within a <a > element
@@ -1438,13 +1482,21 @@ final class TextExtractor {
if (groupState.italic) {
start("i");
}
+ if (groupState.vert != 0) {
+ start(vertTag(groupState.vert));
+ }
}
+
private void endStyles(GroupState groupState) throws TikaException,
IOException, SAXException {
//don't change styles within a <a > element
if (fieldState != 0) {
return;
}
- // Ensure </i></b> order
+ // Ensure </sup|sub></i></b> order
+ if (groupState.vert != 0) {
+ end(vertTag(groupState.vert));
+ groupState.vert = 0;
+ }
if (groupState.italic) {
end("i");
groupState.italic = false;
@@ -1577,6 +1629,13 @@ final class TextExtractor {
}
//only modify styles if we're not in a hyperlink
if (fieldState == 0) {
+ // Close sup/sub if it or anything outside it changes
+ if (groupState.vert != 0 && (groupState.vert !=
outerGroupState.vert
+ || groupState.italic != outerGroupState.italic
+ || groupState.bold != outerGroupState.bold)) {
+ end(vertTag(groupState.vert));
+ groupState.vert = 0;
+ }
// Close italic, if outer does not have italic or
// bold changed:
if (groupState.italic) {
@@ -1600,6 +1659,10 @@ final class TextExtractor {
if (!groupState.italic && outerGroupState.italic) {
start("i");
}
+
+ if (groupState.vert == 0 && outerGroupState.vert != 0) {
+ start(vertTag(outerGroupState.vert));
+ }
}
groupState = outerGroupState;
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
index 54f1dc737c..aaba2e2c96 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/WordParserTest.java
@@ -221,7 +221,8 @@ public class WordParserTest extends TikaTest {
assertTrue(xml.contains("<b>Bold</b>"));
assertTrue(xml.contains("<i>italic</i>"));
assertTrue(xml.contains("<u>underline</u>"));
-
+ assertContains("<sup>superscript</sup>", xml);
+ assertContains("<sub>subscript", xml);
}
@Test
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLDocxSAXTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLDocxSAXTest.java
index aa0bdb7608..aaf20a109c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLDocxSAXTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLDocxSAXTest.java
@@ -527,7 +527,9 @@ public class OOXMLDocxSAXTest extends TikaTest {
assertContains("<b>Bold</b>", xml);
assertContains("<i>italic</i>", xml);
assertContains("<u>underline</u>", xml);
- assertContains("<s>strikethrough</s>", xml);
+ // the run is also subscript
+ assertContains("<s><sub>strikethrough</sub></s>", xml);
+ assertContains("<sup>superscript</sup>", xml);
}
@Test
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPptxSAXTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPptxSAXTest.java
index 069a3ab6e6..f9921d5e03 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPptxSAXTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLPptxSAXTest.java
@@ -153,12 +153,12 @@ public class OOXMLPptxSAXTest extends TikaTest {
Metadata metadata = new Metadata();
String xml = getXML("testPPT_various.pptx", metadata).xml;
assertContains("<p>Footnote appears here", xml);
- assertContains("<p>[1] This is a footnote.", xml);
+ assertContains("<p><sup>[1]</sup> This is a footnote.", xml);
assertContains("<p>This is the header text.</p>", xml);
assertContains("<p>This is the footer text.</p>", xml);
assertContains("<p>Here is a text box</p>", xml);
assertContains("<p>Bold", xml);
- assertContains("italic underline superscript subscript", xml);
+ assertContains("italic underline <sup>superscript</sup>
<sub>subscript</sub>", xml);
assertContains("<p>Here is a citation:", xml);
assertContains("Figure 1 This is a caption for Figure 1", xml);
assertContains("(Kramer)", xml);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLSupSubTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLSupSubTest.java
new file mode 100644
index 0000000000..95d10488f4
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/ooxml/OOXMLSupSubTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.microsoft.ooxml;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayOutputStream;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.junit.jupiter.api.Test;
+import org.xml.sax.helpers.DefaultHandler;
+
+import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.utils.XMLReaderUtils;
+
+public class OOXMLSupSubTest extends TikaTest {
+
+ private static final String W =
+
"xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"";;
+
+ @Test
+ public void testDocx() throws Exception {
+ String styles = "<w:styles " + W + "><w:style w:type=\"character\"
w:styleId=\"Note\">"
+ + "<w:name w:val=\"Note Mark\"/><w:rPr><w:vertAlign
w:val=\"superscript\"/>"
+ + "</w:rPr></w:style></w:styles>";
+ String body = run("E = mc", "") + run("2", "<w:vertAlign
w:val=\"superscript\"/>")
+ + run(" and H", "") + run("2", "<w:vertAlign
w:val=\"subscript\"/>")
+ + run("O, note", "") + run("3", "<w:rStyle w:val=\"Note\"/>")
+ + run(" bold", "<w:b/>")
+ + run("x", "<w:b/><w:vertAlign w:val=\"superscript\"/>")
+ + run("y", "<w:vertAlign w:val=\"superscript\"/>")
+ // a direct baseline overrides the style
+ + run(" 4", "<w:rStyle w:val=\"Note\"/><w:vertAlign
w:val=\"baseline\"/>");
+ String xml = parse(docx("<w:document " + W + "><w:body><w:p>" + body
+ + "</w:p></w:body></w:document>", styles));
+ assertContains("E = mc<sup>2</sup> and H<sub>2</sub>O,
note<sup>3</sup>", xml);
+ // sup nests inside b, and closes before </b>
+ assertContains("<b> bold<sup>x</sup></b><sup>y</sup>", xml);
+ assertNotContained("<sup> 4", xml);
+ }
+
+ @Test
+ public void testPptx() throws Exception {
+ String xml = parse(pptx("<a:r><a:rPr lang=\"en-US\"/><a:t>E =
mc</a:t></a:r>"
+ + "<a:r><a:rPr lang=\"en-US\"
baseline=\"30000\"/><a:t>2</a:t></a:r>"
+ + "<a:r><a:rPr lang=\"en-US\"/><a:t> and H</a:t></a:r>"
+ + "<a:r><a:rPr lang=\"en-US\"
baseline=\"-25000\"/><a:t>2</a:t></a:r>"
+ + "<a:r><a:rPr lang=\"en-US\"
baseline=\"0\"/><a:t>O</a:t></a:r>"));
+ assertContains("E = mc<sup>2</sup> and H<sub>2</sub>O", xml);
+ }
+
+ private static String run(String text, String rPr) {
+ return "<w:r>" + (rPr.isEmpty() ? "" : "<w:rPr>" + rPr + "</w:rPr>")
+ + "<w:t xml:space=\"preserve\">" + text + "</w:t></w:r>";
+ }
+
+ private String parse(byte[] bytes) throws Exception {
+ String xml;
+ try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+ xml = getXML(tis, AUTO_DETECT_PARSER, new Metadata()).xml;
+ }
+ XMLReaderUtils.parseSAX(TikaInputStream.get(xml.getBytes(UTF_8)), new
DefaultHandler(),
+ new ParseContext());
+ return xml;
+ }
+
+ private static byte[] docx(String document, String styles) throws
Exception {
+ Map<String, String> parts = new LinkedHashMap<>();
+ parts.put("[Content_Types].xml", "<Types
xmlns=\"http://schemas.openxmlformats.org/";
+ + "package/2006/content-types\">"
+ + "<Default Extension=\"rels\"
ContentType=\"application/vnd.openxmlformats-"
+ + "package.relationships+xml\"/>"
+ + "<Default Extension=\"xml\"
ContentType=\"application/xml\"/>"
+ + "<Override PartName=\"/word/document.xml\"
ContentType=\"application/"
+ +
"vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml\"/>"
+ + "<Override PartName=\"/word/styles.xml\"
ContentType=\"application/"
+ +
"vnd.openxmlformats-officedocument.wordprocessingml.styles+xml\"/>"
+ + "</Types>");
+ parts.put("_rels/.rels", rels("rId1",
"http://schemas.openxmlformats.org/officeDocument/";
+ + "2006/relationships/officeDocument", "word/document.xml"));
+ parts.put("word/_rels/document.xml.rels", rels("rId1",
+
"http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles";,
+ "styles.xml"));
+ parts.put("word/document.xml", document);
+ parts.put("word/styles.xml", styles);
+ return zip(parts);
+ }
+
+ private static byte[] pptx(String runs) throws Exception {
+ String pml =
"http://schemas.openxmlformats.org/presentationml/2006/main";;
+ String dml = "http://schemas.openxmlformats.org/drawingml/2006/main";;
+ String rel =
"http://schemas.openxmlformats.org/officeDocument/2006/relationships";;
+ Map<String, String> parts = new LinkedHashMap<>();
+ parts.put("[Content_Types].xml", "<Types
xmlns=\"http://schemas.openxmlformats.org/";
+ + "package/2006/content-types\">"
+ + "<Default Extension=\"rels\"
ContentType=\"application/vnd.openxmlformats-"
+ + "package.relationships+xml\"/>"
+ + "<Default Extension=\"xml\"
ContentType=\"application/xml\"/>"
+ + "<Override PartName=\"/ppt/presentation.xml\"
ContentType=\"application/"
+ +
"vnd.openxmlformats-officedocument.presentationml.presentation.main+xml\"/>"
+ + "<Override PartName=\"/ppt/slides/slide1.xml\"
ContentType=\"application/"
+ +
"vnd.openxmlformats-officedocument.presentationml.slide+xml\"/>"
+ + "</Types>");
+ parts.put("_rels/.rels", rels("rId1", rel + "/officeDocument",
"ppt/presentation.xml"));
+ parts.put("ppt/_rels/presentation.xml.rels", rels("rId2", rel +
"/slide",
+ "slides/slide1.xml"));
+ parts.put("ppt/presentation.xml", "<p:presentation xmlns:p=\"" + pml +
"\" xmlns:r=\""
+ + rel + "\"><p:sldIdLst><p:sldId id=\"256\"
r:id=\"rId2\"/></p:sldIdLst>"
+ + "</p:presentation>");
+ parts.put("ppt/slides/slide1.xml", "<p:sld xmlns:p=\"" + pml + "\"
xmlns:a=\"" + dml
+ + "\"><p:cSld><p:spTree><p:sp><p:txBody><a:bodyPr/><a:p>" +
runs
+ + "</a:p></p:txBody></p:sp></p:spTree></p:cSld></p:sld>");
+ return zip(parts);
+ }
+
+ private static String rels(String id, String type, String target) {
+ return "<Relationships
xmlns=\"http://schemas.openxmlformats.org/package/2006/";
+ + "relationships\"><Relationship Id=\"" + id + "\" Type=\"" +
type
+ + "\" Target=\"" + target + "\"/></Relationships>";
+ }
+
+ private static byte[] zip(Map<String, String> parts) throws Exception {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try (ZipOutputStream zos = new ZipOutputStream(bos)) {
+ for (Map.Entry<String, String> e : parts.entrySet()) {
+ zos.putNextEntry(new ZipEntry(e.getKey()));
+ zos.write(e.getValue().getBytes(UTF_8));
+ zos.closeEntry();
+ }
+ }
+ return bos.toByteArray();
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
index e9c5950027..ba14069a0e 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-microsoft-module/src/test/java/org/apache/tika/parser/microsoft/rtf/RTFParserTest.java
@@ -46,6 +46,7 @@ import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.utils.XMLReaderUtils;
/**
* Junit test class for the Tika {@link RTFParser}
@@ -554,4 +555,37 @@ public class RTFParserTest extends TikaTest {
assertContains("this Agreement on today",
getXML("testRTFTIKA_2899.rtf").xml);
}
+
+ private String rtfXml(String body) throws Exception {
+ String rtf = "{\\rtf1\\ansi\\deff0{\\fonttbl{\\f0 Times;}}\n" + body +
"\n}";
+ String xml;
+ try (TikaInputStream tis =
TikaInputStream.get(rtf.getBytes(StandardCharsets.US_ASCII))) {
+ xml = getXML(tis, new RTFParser(), new Metadata()).xml;
+ }
+ // well-formed: sup/sub must nest inside b/i on every path
+
XMLReaderUtils.parseSAX(TikaInputStream.get(xml.getBytes(StandardCharsets.UTF_8)),
+ new DefaultHandler(), new ParseContext());
+ return xml;
+ }
+
+ @Test
+ public void testSupSub() throws Exception {
+ assertContains("<p>E = mc<sup>2</sup> and H<sub>2</sub>O,
note<sup>3</sup></p>",
+ rtfXml("\\pard E = mc{\\super 2} and H{\\sub 2}O, note{\\super
3}\\par"));
+ // toggled without groups
+ assertContains("<p>x<sup>2</sup>+y<sub>i</sub>z</p>",
+ rtfXml("\\pard x\\super 2\\nosupersub +y\\sub i\\plain
z\\par"));
+ }
+
+ @Test
+ public void testSupSubNestsInsideBoldItalic() throws Exception {
+ // sup opened before bold, bold toggled inside it, italic around it
+ String xml = rtfXml("\\pard a{\\super b\\b c\\b0 d}{\\i e{\\super
f}g}\\par"
+ + "\\pard{\\b\\super h\\par i}\\par");
+ assertContains("a<sup>b</sup><b><sup>c</sup></b><sup>d</sup>", xml);
+ assertContains("<i>e<sup>f</sup>g</i>", xml);
+ // carried across \\par with the bold around it
+ assertContains("<b><sup>h</sup></b></p>", xml);
+ assertContains("<b><sup>i</sup></b>", xml);
+ }
}
