This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new 742b468ce3 TIKA-5014: write dupes up to a point (#3463)
742b468ce3 is described below
commit 742b468ce337897b60ea20954735c399a15fa265
Author: Tim Allison <[email protected]>
AuthorDate: Fri Oct 30 17:07:11 2026 -0400
TIKA-5014: write dupes up to a point (#3463)
* TIKA-5014: write dupes up to a point
* TIKA-5014: update ODS macro tests for dropped empty rows
---
CHANGES.txt | 7 +
.../org/apache/tika/parser/odf/ODFParserTest.java | 6 +-
.../tika/parser/odf/FlatOpenDocumentParser.java | 17 +-
.../tika/parser/odf/OpenDocumentBodyHandler.java | 67 ++++-
.../tika/parser/odf/OpenDocumentContentParser.java | 6 +-
.../tika/parser/odf/RepeatedElementHandler.java | 271 +++++++++++++++++++++
.../parser/odf/OpenDocumentContentParserTest.java | 102 ++++++++
7 files changed, 453 insertions(+), 23 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 77e7c60b46..1846044090 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -56,6 +56,13 @@ Release 4.3.0 - ???
and split GFM table cells on |. Math is emitted as <span class="math
inline"> or
<span class="math display">, and the Markdown writer outputs it unescaped
(TIKA-5011).
+ * The OpenDocument parser writes repeated cells and rows
(table:number-columns-repeated,
+ table:number-rows-repeated) once per repeat; a cell was written once with
that
+ colspan and a row once, so values repeated across columns or rows were
lost. Empty
+ cells and rows at the end of a row or table, such as the filler out to a
sheet's
+ last column and row, are dropped. Past 10 million characters of repeated
content
+ per document the rest are written once, with a tk:exception:warn entry
(TIKA-5014).
+
* VLM text recognizers keep superscripts and subscripts: inline <sup>/<sub>
in model
output (x<sup>2</sup>, H<sub>2</sub>O, footnote markers) reach the XHTML
as sup/sub
instead of plain text ("x2"), and the Markdown writer outputs them as
inline HTML
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
index c2b98b799e..c2b8f82842 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
@@ -91,7 +91,8 @@ public class ODFParserTest extends TikaTest {
assertEquals(4, metadataList.size());
Metadata parent = metadataList.get(0);
- assertContains("<tr>", parent.get(TikaCoreProperties.TIKA_CONTENT));
+ // the sheet is an image shape and one empty row, which is dropped
(TIKA-5014)
+ assertContains("<table><img",
parent.get(TikaCoreProperties.TIKA_CONTENT));
assertEquals("application/vnd.oasis.opendocument.spreadsheet",
parent.get(HttpHeaders.CONTENT_TYPE));
@@ -169,7 +170,8 @@ public class ODFParserTest extends TikaTest {
assertEquals(3, metadataList.size());
Metadata parent = metadataList.get(0);
- assertContains("<tr>", parent.get(TikaCoreProperties.TIKA_CONTENT));
+ // the sheet is an image shape and one empty row, which is dropped
(TIKA-5014)
+ assertContains("<table><img",
parent.get(TikaCoreProperties.TIKA_CONTENT));
assertEquals("application/vnd.oasis.opendocument.flat.spreadsheet",
parent.get(HttpHeaders.CONTENT_TYPE));
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
index 9f391a6e83..26a31af9b5 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
@@ -110,13 +110,14 @@ public class FlatOpenDocumentParser implements Parser {
// and masks the real well-formedness error.
final XHTMLBalancingHandler balancer = new
XHTMLBalancingHandler(xhtml);
tis.setCloseShield();
+ FlatOpenDocumentParserHandler fodHandler = null;
try {
- ContentHandler fodHandler = getContentHandler(balancer, metadata,
context);
+ fodHandler = getContentHandler(balancer, metadata, context);
XMLReaderUtils.parseSAX(tis,
new EmbeddedContentHandler(fodHandler), context);
//can only detect subtype (text/pres/sheet) during parse.
//update it here.
- MediaType detected = ((FlatOpenDocumentParserHandler)
fodHandler).getDetectedType();
+ MediaType detected = fodHandler.getDetectedType();
if (detected != null) {
metadata.set(HttpHeaders.CONTENT_TYPE, detected.toString());
}
@@ -125,6 +126,9 @@ public class FlatOpenDocumentParser implements Parser {
// loop. Without this, the finally's xhtml.endDocument() throws
// on the unbalanced stack and the masked </body> vs <p>/<b>
// hides the real failure (truncated input, malformed XML, etc.).
+ if (fodHandler != null) {
+ fodHandler.bodyHandler.flushBuffered(e);
+ }
balancer.drainOpenElements();
throw e;
} finally {
@@ -140,8 +144,9 @@ public class FlatOpenDocumentParser implements Parser {
public boolean isExtractMacros() {
return extractMacros;
}
- private ContentHandler getContentHandler(ContentHandler handler, Metadata
metadata,
- ParseContext context) {
+ private FlatOpenDocumentParserHandler getContentHandler(ContentHandler
handler,
+ Metadata metadata,
+ ParseContext
context) {
return new FlatOpenDocumentParserHandler(handler, metadata, context,
extractMacros);
}
@@ -154,7 +159,7 @@ public class FlatOpenDocumentParser implements Parser {
private final ContentHandler defaultHandler = new DefaultHandler();
- private final ContentHandler bodyHandler;
+ private final OpenDocumentBodyHandler bodyHandler;
private final ContentHandler metadataHandler;
private final ContentHandler macroHandler;
private final boolean extractMacros;
@@ -180,7 +185,7 @@ public class FlatOpenDocumentParser implements Parser {
this.extractMacros = extractMacros;
this.bodyHandler = new OpenDocumentBodyHandler(new
NSNormalizerContentHandler(baseHandler),
- parseContext);
+ parseContext, metadata);
this.metadataHandler = new NSNormalizerContentHandler(
OpenDocumentMetaParser.getContentHandler(metadata,
parseContext));
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
index 463d212f0a..3435dbfd2a 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
@@ -72,6 +72,9 @@ class OpenDocumentBodyHandler extends
ElementMappingContentHandler {
new HashMap<>();
private static final char[] SPACE = new char[]{' '};
private static final String CLASS = "class";
+ // ODF 1.2+ sheet limits
+ private static final int MAX_COLUMNS = 16384;
+ private static final int MAX_ROWS = 1048576;
private static final Attributes ANNOTATION_ATTRIBUTES =
buildAttributes(CLASS, "annotation");
private static final Attributes NOTE_ATTRIBUTES = buildAttributes(CLASS,
"note");
private static final Attributes NOTES_ATTRIBUTES = buildAttributes(CLASS,
"notes");
@@ -104,27 +107,21 @@ class OpenDocumentBodyHandler extends
ElementMappingContentHandler {
// create HTML tables from table:-tags
MAPPINGS.put(new QName(TABLE_NS, "table"), new TargetElement(XHTML,
"table"));
- // repeating of rows is ignored; for columns, see below!
+ // number-rows-repeated: see RepeatedElementHandler
MAPPINGS.put(new QName(TABLE_NS, "table-row"), new
TargetElement(XHTML, "tr"));
// special mapping for rowspan/colspan attributes
final HashMap<QName, QName> tableCellAttsMapping = new HashMap<>();
tableCellAttsMapping
.put(new QName(TABLE_NS, "number-columns-spanned"), new
QName("colspan"));
tableCellAttsMapping.put(new QName(TABLE_NS, "number-rows-spanned"),
new QName("rowspan"));
- /* TODO: The following is not correct, the cell should be repeated not
spanned!
- * Code generates a HTML cell, spanning all repeated columns, to make
the cell look correct.
- * Problems may occur when both spanning and repeating is given, which
is not allowed by
- * spec.
- * Cell spanning instead of repeating is not a problem, because
OpenOffice uses it
- * only for empty cells.
- */
- tableCellAttsMapping
- .put(new QName(TABLE_NS, "number-columns-repeated"), new
QName("colspan"));
+ // number-columns-repeated: see RepeatedElementHandler
MAPPINGS.put(new QName(TABLE_NS, "table-cell"),
new TargetElement(XHTML, "td", tableCellAttsMapping));
}
private final ContentHandler handler;
+ private final RepeatedElementHandler repeatedCells;
+ private final RepeatedElementHandler repeatedRows;
private final ParseContext parseContext;
private final BitSet textNodeStack = new BitSet();
//have we written the start style tags
@@ -156,12 +153,49 @@ class OpenDocumentBodyHandler extends
ElementMappingContentHandler {
// is that bold/italic/underline runs that begin or end inside a link lose
// their formatting in the captured XHTML.
private int anchorDepth = 0;
- OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext)
{
- super(handler, MAPPINGS);
- this.handler = handler;
+ OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext,
Metadata metadata) {
+ this(new RepeatedElementHandler(handler, "tr", "table", MAX_ROWS,
+ new RepeatedElementHandler.Budget(metadata)), parseContext);
+ }
+
+ private OpenDocumentBodyHandler(RepeatedElementHandler rows, ParseContext
parseContext) {
+ this(new RepeatedElementHandler(rows, "td", "tr", MAX_COLUMNS,
rows.budget()), rows,
+ parseContext);
+ }
+
+ private OpenDocumentBodyHandler(RepeatedElementHandler cells,
RepeatedElementHandler rows,
+ ParseContext parseContext) {
+ super(cells, MAPPINGS);
+ this.handler = cells;
+ this.repeatedCells = cells;
+ this.repeatedRows = rows;
this.parseContext = parseContext;
}
+ /**
+ * Writes table content still held for repeat handling. Call when the
parse fails,
+ * so the partly read row is not lost.
+ */
+ void flushBuffered(Exception cause) {
+ try {
+ repeatedCells.flushBuffered();
+ repeatedRows.flushBuffered();
+ } catch (SAXException e) {
+ cause.addSuppressed(e);
+ }
+ }
+
+ private static int parseRepeat(String value) {
+ if (value == null) {
+ return 1;
+ }
+ try {
+ return Math.max(1, Integer.parseInt(value.trim()));
+ } catch (NumberFormatException e) {
+ return 1;
+ }
+ }
+
private static Attributes buildAttributes(String key, String value) {
AttributesImpl attrs = new AttributesImpl();
attrs.addAttribute("", key, key, "CDATA", value);
@@ -478,6 +512,13 @@ class OpenDocumentBodyHandler extends
ElementMappingContentHandler {
}
// call next handler if no filtering
if (completelyFiltered == 0) {
+ if (TABLE_NS.equals(namespaceURI) &&
"table-cell".equals(localName)) {
+ repeatedCells.repeatNext(
+ parseRepeat(attrs.getValue(TABLE_NS,
"number-columns-repeated")));
+ } else if (TABLE_NS.equals(namespaceURI) &&
"table-row".equals(localName)) {
+ repeatedRows.repeatNext(
+ parseRepeat(attrs.getValue(TABLE_NS,
"number-rows-repeated")));
+ }
// special handling of text:h, that are directly passed
// to incoming handler
if (TEXT_NS.equals(namespaceURI) && "h".equals(localName)) {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
index a9646265e0..8fce0ef133 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
@@ -22,7 +22,6 @@ import java.util.Set;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.exception.TikaException;
import org.apache.tika.io.TikaInputStream;
@@ -50,12 +49,15 @@ public class OpenDocumentContentParser implements Parser {
void parseInternal(TikaInputStream tis, final ContentHandler handler,
Metadata metadata,
ParseContext context) throws IOException, SAXException,
TikaException {
- DefaultHandler dh = new OpenDocumentBodyHandler(handler, context);
+ OpenDocumentBodyHandler dh = new OpenDocumentBodyHandler(handler,
context, metadata);
tis.setCloseShield();
try {
XMLReaderUtils.parseSAX(tis,
new NSNormalizerContentHandler(dh), context);
+ } catch (SAXException | IOException | TikaException e) {
+ dh.flushBuffered(e);
+ throw e;
} finally {
tis.removeCloseShield();
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
new file mode 100644
index 0000000000..17b5ba6ec6
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.odf;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.AttributesImpl;
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.sax.ContentHandlerDecorator;
+
+/**
+ * Writes an ODF table cell or row that carries a repeat count
+ * ({@code table:number-columns-repeated}, {@code table:number-rows-repeated})
once per
+ * repeat.
+ * <p>
+ * ODF stores runs of identical cells or rows once with a count. One with
content is
+ * replayed. Empty ones, repeated or not, are held back and written only if a
sibling
+ * with content follows before the parent closes, so the filler that pads a
sheet out to
+ * its last column or row (often thousands of cells, a million rows) is
dropped.
+ */
+class RepeatedElementHandler extends ContentHandlerDecorator {
+
+ /** Characters a document may add by replaying, beyond each element's
first copy. */
+ static final long MAX_REPLAYED_CHARS = 10_000_000L;
+
+ /** Replay budget shared by the cell and row handlers of one document. */
+ static final class Budget {
+ private final Metadata metadata;
+ private long replayedChars;
+ private boolean warned;
+
+ Budget(Metadata metadata) {
+ this.metadata = metadata;
+ }
+
+ /** Whether {@code extra} more characters fit; warns once when they
don't. */
+ boolean take(long extra) {
+ if (replayedChars + extra <= MAX_REPLAYED_CHARS) {
+ replayedChars += extra;
+ return true;
+ }
+ if (!warned && metadata != null) {
+ metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
+ "OpenDocument repeated cells/rows exceeded " +
MAX_REPLAYED_CHARS
+ + " replayed characters; the rest were written
once");
+ }
+ warned = true;
+ return false;
+ }
+ }
+
+ /** A run of held-back empty elements sharing a start tag. */
+ private static final class EmptyRun {
+ final String uri;
+ final String localName;
+ final String name;
+ final Attributes atts;
+ int count;
+
+ EmptyRun(String uri, String localName, String name, Attributes atts,
int count) {
+ this.uri = uri;
+ this.localName = localName;
+ this.name = name;
+ this.atts = atts;
+ this.count = count;
+ }
+ }
+
+ private interface Event {
+ void replay(ContentHandler h) throws SAXException;
+ }
+
+ private final ContentHandler out;
+ private final String element;
+ private final String parent;
+ private final int maxRepeat;
+ private final Budget budget;
+
+ private int nextRepeat;
+ private List<Event> buffer;
+ private String bufferUri;
+ private String bufferName;
+ private Attributes bufferAtts;
+ private int depth;
+ private int repeat;
+ private boolean hasContent;
+ private long chars;
+ private final Deque<EmptyRun> pendingEmpty = new ArrayDeque<>();
+ private long pendingCount;
+
+ /**
+ * @param element the XHTML element that repeats ({@code td} or {@code tr})
+ * @param parent the element whose end drops pending empty copies
+ */
+ RepeatedElementHandler(ContentHandler out, String element, String parent,
int maxRepeat,
+ Budget budget) {
+ super(out);
+ this.out = out;
+ this.element = element;
+ this.parent = parent;
+ this.maxRepeat = maxRepeat;
+ this.budget = budget;
+ }
+
+ /** After a parse failure: writes a buffered element as far as it got,
once. */
+ void flushBuffered() throws SAXException {
+ if (buffer != null) {
+ List<Event> partial = buffer;
+ buffer = null;
+ for (Event e : partial) {
+ e.replay(out);
+ }
+ }
+ }
+
+ Budget budget() {
+ return budget;
+ }
+
+ /** Called before the start of an element whose repeat count is {@code n}.
*/
+ void repeatNext(int n) {
+ if (buffer == null) {
+ nextRepeat = Math.min(n, maxRepeat);
+ }
+ }
+
+ @Override
+ public void startElement(String uri, String localName, String name,
Attributes atts)
+ throws SAXException {
+ if (buffer != null) {
+ depth++;
+ if (localName.equals("img")) {
+ hasContent = true;
+ }
+ AttributesImpl copy = new AttributesImpl(atts);
+ buffer.add(h -> h.startElement(uri, localName, name, copy));
+ return;
+ }
+ if (localName.equals(element)) {
+ buffer = new ArrayList<>();
+ depth = 0;
+ repeat = Math.max(1, nextRepeat);
+ hasContent = false;
+ chars = 0;
+ nextRepeat = 0;
+ AttributesImpl copy = new AttributesImpl(atts);
+ bufferUri = uri;
+ bufferName = name;
+ bufferAtts = copy;
+ buffer.add(h -> h.startElement(uri, localName, name, copy));
+ return;
+ } else if (localName.equals(parent)) {
+ clearPending();
+ }
+ nextRepeat = 0;
+ super.startElement(uri, localName, name, atts);
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String name) throws
SAXException {
+ if (buffer != null) {
+ buffer.add(h -> h.endElement(uri, localName, name));
+ if (depth-- > 0) {
+ return;
+ }
+ List<Event> done = buffer;
+ buffer = null;
+ if (!hasContent) {
+ hold(bufferUri, localName, bufferName, bufferAtts, repeat);
+ return;
+ }
+ writePendingEmpty();
+ int copies = repeat;
+ if (copies > 1 && !budget.take(chars * (copies - 1))) {
+ copies = 1;
+ }
+ for (int i = 0; i < copies; i++) {
+ for (Event e : done) {
+ e.replay(out);
+ }
+ }
+ return;
+ }
+ if (localName.equals(parent)) {
+ clearPending();
+ }
+ super.endElement(uri, localName, name);
+ }
+
+ @Override
+ public void characters(char[] ch, int start, int length) throws
SAXException {
+ if (buffer != null) {
+ char[] copy = new char[length];
+ System.arraycopy(ch, start, copy, 0, length);
+ if (!hasContent) {
+ for (char c : copy) {
+ if (!Character.isWhitespace(c)) {
+ hasContent = true;
+ break;
+ }
+ }
+ }
+ chars += length;
+ buffer.add(h -> h.characters(copy, 0, copy.length));
+ return;
+ }
+ super.characters(ch, start, length);
+ }
+
+ @Override
+ public void ignorableWhitespace(char[] ch, int start, int length) throws
SAXException {
+ if (buffer != null) {
+ char[] copy = new char[length];
+ System.arraycopy(ch, start, copy, 0, length);
+ buffer.add(h -> h.ignorableWhitespace(copy, 0, copy.length));
+ return;
+ }
+ super.ignorableWhitespace(ch, start, length);
+ }
+
+ private void hold(String uri, String localName, String name, Attributes
atts, int count) {
+ int n = (int) Math.min(count, maxRepeat - pendingCount);
+ if (n <= 0) {
+ return;
+ }
+ EmptyRun last = pendingEmpty.peekLast();
+ if (last != null && last.atts.getLength() == 0 && atts.getLength() ==
0) {
+ last.count += n;
+ } else {
+ pendingEmpty.addLast(new EmptyRun(uri, localName, name, atts, n));
+ }
+ pendingCount += n;
+ }
+
+ private void clearPending() {
+ pendingEmpty.clear();
+ pendingCount = 0;
+ }
+
+ private void writePendingEmpty() throws SAXException {
+ for (EmptyRun run : pendingEmpty) {
+ for (int i = 0; i < run.count; i++) {
+ super.startElement(run.uri, run.localName, run.name, run.atts);
+ super.endElement(run.uri, run.localName, run.name);
+ }
+ }
+ clearPending();
+ }
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
index 8acf8c4ba7..5a0b87f7ef 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
@@ -16,7 +16,9 @@
*/
package org.apache.tika.parser.odf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.StandardCharsets;
@@ -25,6 +27,7 @@ import java.util.regex.Pattern;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.junit.jupiter.api.Test;
+import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.TikaTest;
@@ -32,6 +35,7 @@ import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ToXMLContentHandler;
import org.apache.tika.utils.XMLReaderUtils;
public class OpenDocumentContentParserTest extends TikaTest {
@@ -68,4 +72,102 @@ public class OpenDocumentContentParserTest extends TikaTest
{
XMLReaderUtils.parseSAX(TikaInputStream.get(xml.getBytes(StandardCharsets.UTF_8)),
new DefaultHandler(), new ParseContext());
}
+
+ @Test
+ public void testRepeatedCells() throws Exception {
+ String content = "<office:document-content"
+ + "
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+ + "
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+ + "
xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\">"
+ + "<office:body><office:spreadsheet><table:table
table:name=\"S\">"
+ + "<table:table-row>"
+ + "<table:table-cell
table:number-columns-repeated=\"3\"><text:p>TEST</text:p>"
+ + "</table:table-cell>"
+ + "<table:table-cell table:number-columns-repeated=\"1020\"/>"
+ + "</table:table-row><table:table-row>"
+ + "<table:table-cell><text:p>a</text:p></table:table-cell>"
+ + "<table:table-cell table:number-columns-repeated=\"2\"/>"
+ + "<table:table-cell><text:p>b</text:p></table:table-cell>"
+ + "<table:table-cell table:number-columns-repeated=\"1000\"/>"
+ + "</table:table-row><table:table-row>"
+ + "<table:table-cell
table:number-columns-spanned=\"2\"><text:p>wide</text:p>"
+ + "</table:table-cell><table:covered-table-cell/>"
+ + "<table:table-cell><text:p>c</text:p></table:table-cell>"
+ + "</table:table-row>"
+ // a repeated row with content, empty rows between values,
trailing filler
+ + "<table:table-row table:number-rows-repeated=\"2\">"
+ + "<table:table-cell><text:p>r</text:p></table:table-cell>"
+ + "<table:table-cell
table:number-columns-repeated=\"2\"><text:p>s</text:p>"
+ + "</table:table-cell></table:table-row>"
+ + "<table:table-row table:number-rows-repeated=\"2\">"
+ + "<table:table-cell
table:number-columns-repeated=\"1024\"/></table:table-row>"
+ +
"<table:table-row><table:table-cell><text:p>z</text:p></table:table-cell>"
+ + "</table:table-row>"
+ + "<table:table-row table:number-rows-repeated=\"1048000\">"
+ + "<table:table-cell
table:number-columns-repeated=\"1024\"/></table:table-row>"
+ // a plain empty row after the filler must not bring it back
+ +
"<table:table-row><table:table-cell/><table:table-cell/></table:table-row>"
+ +
"</table:table></office:spreadsheet></office:body></office:document-content>";
+ String xml;
+ try (TikaInputStream tis =
TikaInputStream.get(content.getBytes(StandardCharsets.UTF_8))) {
+ xml = getXML(tis, new OpenDocumentContentParser(), new
Metadata()).xml;
+ }
+ String table = xml.substring(xml.indexOf("<table>"),
xml.indexOf("</table>"))
+ .replaceAll("\\s+", "");
+
assertEquals("<table><tr><td><p>TEST</p></td><td><p>TEST</p></td><td><p>TEST</p></td></tr>"
+ + "<tr><td><p>a</p></td><td/><td/><td><p>b</p></td></tr>"
+ + "<tr><tdcolspan=\"2\"><p>wide</p></td><td><p>c</p></td></tr>"
+ +
"<tr><td><p>r</p></td><td><p>s</p></td><td><p>s</p></td></tr>"
+ +
"<tr><td><p>r</p></td><td><p>s</p></td><td><p>s</p></td></tr>"
+ + "<tr/><tr/><tr><td><p>z</p></td></tr>", table);
+ }
+
+ @Test
+ public void testRepeatedCellBudget() throws Exception {
+ String big = "x".repeat(1000);
+ String content = "<office:document-content"
+ + "
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+ + "
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+ + "
xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\">"
+ + "<office:body><office:spreadsheet><table:table
table:name=\"S\">"
+ + "<table:table-row><table:table-cell
table:number-columns-spanned=\"2\"/>"
+ +
"<table:covered-table-cell/><table:table-cell><text:p>v</text:p>"
+ + "</table:table-cell></table:table-row>"
+ + "<table:table-row><table:table-cell
table:number-columns-repeated=\"16384\">"
+ + "<text:p>" + big +
"</text:p></table:table-cell></table:table-row>"
+ +
"</table:table></office:spreadsheet></office:body></office:document-content>";
+ Metadata metadata = new Metadata();
+ String xml;
+ try (TikaInputStream tis =
TikaInputStream.get(content.getBytes(StandardCharsets.UTF_8))) {
+ xml = getXML(tis, new OpenDocumentContentParser(), metadata).xml;
+ }
+ // an empty merged cell keeps its span when a later cell is written
+ assertContains("<td colspan=\"2\" />", xml);
+ assertEquals(1, xml.split(big, -1).length - 1);
+ assertContains("written once",
+
metadata.get(org.apache.tika.metadata.TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+ }
+
+ @Test
+ public void testRowInProgressKeptOnParseFailure() throws Exception {
+ String content = "<office:document-content"
+ + "
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+ + "
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+ + "
xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\">"
+ + "<office:body><office:spreadsheet><table:table
table:name=\"S\">"
+ +
"<table:table-row><table:table-cell><text:p>first</text:p></table:table-cell>"
+ + "</table:table-row><table:table-row>"
+ + "<table:table-cell><text:p>kept</text:p></table:table-cell>"
+ // duplicate attribute: not well-formed
+ + "<table:table-cell office:a=\"1\" office:a=\"2\"/>"
+ +
"</table:table-row></table:table></office:spreadsheet></office:body>"
+ + "</office:document-content>";
+ ToXMLContentHandler out = new ToXMLContentHandler();
+ try (TikaInputStream tis =
TikaInputStream.get(content.getBytes(StandardCharsets.UTF_8))) {
+ assertThrows(SAXException.class, () -> new
OpenDocumentContentParser()
+ .parse(tis, out, new Metadata(), new ParseContext()));
+ }
+ assertContains("first", out.toString());
+ assertContains("kept", out.toString());
+ }
}
