This is an automated email from the ASF dual-hosted git repository.
tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git
The following commit(s) were added to refs/heads/main by this push:
new ed8a8913a2 TIKA-5014: drop empty ODS rows, pad cells only before
content, charge all writes (#3600)
ed8a8913a2 is described below
commit ed8a8913a232395cd764fb6cafae3123f72a193e
Author: Tim Allison <[email protected]>
AuthorDate: Thu Nov 12 14:03:33 2026 -0500
TIKA-5014: drop empty ODS rows, pad cells only before content, charge all
writes (#3600)
---
CHANGES.txt | 14 +-
docs/modules/ROOT/pages/formats.adoc | 18 ++
.../org/apache/tika/parser/odf/ODFParserTest.java | 28 +++
.../configs/tika-config-ods-missing-rows.json | 17 ++
.../tika/parser/odf/FlatOpenDocumentParser.java | 20 +-
.../tika/parser/odf/OpenDocumentBodyHandler.java | 12 +-
.../tika/parser/odf/OpenDocumentContentParser.java | 9 +-
.../apache/tika/parser/odf/OpenDocumentParser.java | 22 ++-
.../tika/parser/odf/RepeatedElementHandler.java | 52 ++++-
.../parser/odf/OpenDocumentContentParserTest.java | 213 ++++++++++++++++++++-
10 files changed, 385 insertions(+), 20 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 0e94fe707e..e524a8437b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -385,10 +385,16 @@ Release 4.3.0 - ???
* The OpenDocument parser writes repeated cells and rows
(table:number-columns-repeated,
table:number-rows-repeated) once per repeat; a cell was written once with
that
- colspan and a row once, so values repeated across columns or rows were
lost. Empty
- cells and rows at the end of a row or table, such as the filler out to a
sheet's
- last column and row, are dropped. Past 10 million characters of repeated
content
- per document the rest are written once, with a tk:exception:warn entry
(TIKA-5014).
+ colspan and a row once, so values repeated across columns or rows were
lost. As in
+ the Excel parsers, empty rows are dropped and empty cells are written
only to pad
+ out to the next value in their row. Past 10 million characters and
elements of
+ padding and repeated content per document, the rest are written once and
padding
+ is dropped, with a tk:exception:warn entry (TIKA-5014).
+
+ * The OpenDocument parsers (open-document-parser,
flat-open-document-parser) take
+ includeMissingRows, as the Excel parsers do: when true, each empty
spreadsheet row
+ before a row with content is written as an empty row, up to 100,000 per
document.
+ The default is false (TIKA-5014).
* VLM text recognizers keep superscripts and subscripts: inline <sup>/<sub>
in model
output (x<sup>2</sup>, H<sub>2</sub>O, footnote markers) reach the XHTML
as sup/sub
diff --git a/docs/modules/ROOT/pages/formats.adoc
b/docs/modules/ROOT/pages/formats.adoc
index fd3f3af91a..0390ff7b9e 100644
--- a/docs/modules/ROOT/pages/formats.adoc
+++ b/docs/modules/ROOT/pages/formats.adoc
@@ -85,6 +85,24 @@ ODF is based;
link:{api}/org/apache/tika/parser/odf/FlatOpenDocumentParser.html[FlatOpenDocumentParser]
handles the single-file flat XML variants (`.fodt`, `.fods`, `.fodp`).
+Both write spreadsheet tables as the Excel parsers do: empty rows are dropped,
+empty cells are written only to pad out to the next value in their row, and
+the filler at the end of a row or sheet is dropped. Set `includeMissingRows`
+to write each empty row before a row with content, as `includeMissingRows`
+does for Excel; at most 100,000 are written per document. Set `extractMacros`
+to extract Basic macros.
+
+[source,json]
+----
+{
+ "parsers": [
+ { "default-parser": {} },
+ { "open-document-parser": { "includeMissingRows": true } },
+ { "flat-open-document-parser": { "includeMissingRows": true } }
+ ]
+}
+----
+
== iWork document formats
The Apple iWork formats (Pages, Numbers, Keynote) are supported by
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
index c2b8f82842..92391130b2 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.parser.odf;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import org.junit.jupiter.api.BeforeAll;
@@ -25,6 +26,7 @@ import org.junit.jupiter.api.Test;
import org.apache.tika.TikaLoaderHelper;
import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.PagedText;
@@ -211,4 +213,30 @@ public class ODFParserTest extends TikaTest {
private void assertImageContentType(String expected, String actual) {
assertEquals(expected, actual);
}
+
+ @Test
+ public void testIncludeMissingRowsConfig() throws Exception {
+ Parser parser =
TikaLoaderHelper.getLoader("tika-config-ods-missing-rows.json")
+ .loadAutoDetectParser();
+ String fods = "<?xml version=\"1.0\"
encoding=\"UTF-8\"?><office:document"
+ + "
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+ + "
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+ + "
xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\""
+ + "
office:mimetype=\"application/vnd.oasis.opendocument.spreadsheet\">"
+ + "<office:body><office:spreadsheet><table:table
table:name=\"S\">"
+ +
"<table:table-row><table:table-cell><text:p>a</text:p></table:table-cell>"
+ + "</table:table-row><table:table-row
table:number-rows-repeated=\"3\">"
+ + "<table:table-cell/></table:table-row>"
+ +
"<table:table-row><table:table-cell><text:p>b</text:p></table:table-cell>"
+ +
"</table:table-row></table:table></office:spreadsheet></office:body>"
+ + "</office:document>";
+ Metadata metadata = new Metadata();
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "rows.fods");
+ String xml;
+ try (TikaInputStream tis =
TikaInputStream.get(fods.getBytes(StandardCharsets.UTF_8))) {
+ xml = getXML(tis, parser, metadata).xml;
+ }
+ assertEquals(3, xml.split("<tr>\\s*<td />\\s*</tr>", -1).length - 1);
+ assertContains("<p>b</p>", xml);
+ }
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/configs/tika-config-ods-missing-rows.json
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/configs/tika-config-ods-missing-rows.json
new file mode 100644
index 0000000000..ae68a2c7de
--- /dev/null
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/configs/tika-config-ods-missing-rows.json
@@ -0,0 +1,17 @@
+{
+ "parsers": [
+ {
+ "default-parser" : {}
+ },
+ {
+ "open-document-parser": {
+ "includeMissingRows": true
+ }
+ },
+ {
+ "flat-open-document-parser": {
+ "includeMissingRows": true
+ }
+ }
+ ]
+}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
index 26a31af9b5..cd665101b3 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
@@ -51,6 +51,7 @@ public class FlatOpenDocumentParser implements Parser {
*/
public static class Config {
public boolean extractMacros = false;
+ public boolean includeMissingRows = false;
}
static final MediaType FLAT_OD =
@@ -68,6 +69,7 @@ public class FlatOpenDocumentParser implements Parser {
.unmodifiableSet(new HashSet<>(Arrays.asList(FLAT_OD, FLAT_ODT,
FLAT_ODP, FLAT_ODS)));
private boolean extractMacros = false;
+ private boolean includeMissingRows = false;
public FlatOpenDocumentParser() {
}
@@ -79,6 +81,7 @@ public class FlatOpenDocumentParser implements Parser {
*/
public FlatOpenDocumentParser(Config config) {
this.extractMacros = config.extractMacros;
+ this.includeMissingRows = config.includeMissingRows;
}
/**
@@ -144,10 +147,20 @@ public class FlatOpenDocumentParser implements Parser {
public boolean isExtractMacros() {
return extractMacros;
}
+
+ /** See {@link OpenDocumentParser#setIncludeMissingRows(boolean)}. */
+ public void setIncludeMissingRows(boolean includeMissingRows) {
+ this.includeMissingRows = includeMissingRows;
+ }
+
+ public boolean isIncludeMissingRows() {
+ return includeMissingRows;
+ }
private FlatOpenDocumentParserHandler getContentHandler(ContentHandler
handler,
Metadata metadata,
ParseContext
context) {
- return new FlatOpenDocumentParserHandler(handler, metadata, context,
extractMacros);
+ return new FlatOpenDocumentParserHandler(handler, metadata, context,
extractMacros,
+ includeMissingRows);
}
private static class FlatOpenDocumentParserHandler extends
ContentHandlerDecorator {
@@ -181,11 +194,12 @@ public class FlatOpenDocumentParser implements Parser {
private int outerSectionDepth = 0;
private FlatOpenDocumentParserHandler(ContentHandler baseHandler,
Metadata metadata,
- ParseContext parseContext,
boolean extractMacros) {
+ ParseContext parseContext,
boolean extractMacros,
+ boolean includeMissingRows) {
this.extractMacros = extractMacros;
this.bodyHandler = new OpenDocumentBodyHandler(new
NSNormalizerContentHandler(baseHandler),
- parseContext, metadata);
+ parseContext, metadata, includeMissingRows);
this.metadataHandler = new NSNormalizerContentHandler(
OpenDocumentMetaParser.getContentHandler(metadata,
parseContext));
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
index 3435dbfd2a..e1fa87fadc 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
@@ -75,6 +75,8 @@ class OpenDocumentBodyHandler extends
ElementMappingContentHandler {
// ODF 1.2+ sheet limits
private static final int MAX_COLUMNS = 16384;
private static final int MAX_ROWS = 1048576;
+ // with includeMissingRows: keeps a 1M-row gap below the zip-bomb guard's
1M-char threshold
+ private static final int MAX_EMPTY_ROWS_WRITTEN = 100_000;
private static final Attributes ANNOTATION_ATTRIBUTES =
buildAttributes(CLASS, "annotation");
private static final Attributes NOTE_ATTRIBUTES = buildAttributes(CLASS,
"note");
private static final Attributes NOTES_ATTRIBUTES = buildAttributes(CLASS,
"notes");
@@ -154,13 +156,19 @@ class OpenDocumentBodyHandler extends
ElementMappingContentHandler {
// their formatting in the captured XHTML.
private int anchorDepth = 0;
OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext,
Metadata metadata) {
+ this(handler, parseContext, metadata, false);
+ }
+
+ OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext,
Metadata metadata,
+ boolean includeMissingRows) {
this(new RepeatedElementHandler(handler, "tr", "table", MAX_ROWS,
+ includeMissingRows ? MAX_EMPTY_ROWS_WRITTEN : 0, "td",
new RepeatedElementHandler.Budget(metadata)), parseContext);
}
private OpenDocumentBodyHandler(RepeatedElementHandler rows, ParseContext
parseContext) {
- this(new RepeatedElementHandler(rows, "td", "tr", MAX_COLUMNS,
rows.budget()), rows,
- parseContext);
+ this(new RepeatedElementHandler(rows, "td", "tr", MAX_COLUMNS,
Long.MAX_VALUE, null,
+ rows.budget()), rows, parseContext);
}
private OpenDocumentBodyHandler(RepeatedElementHandler cells,
RepeatedElementHandler rows,
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
index 8fce0ef133..63028c4031 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
@@ -48,8 +48,15 @@ public class OpenDocumentContentParser implements Parser {
void parseInternal(TikaInputStream tis, final ContentHandler handler,
Metadata metadata,
ParseContext context) throws IOException, SAXException,
TikaException {
+ parseInternal(tis, handler, metadata, context, false);
+ }
+
+ void parseInternal(TikaInputStream tis, final ContentHandler handler,
Metadata metadata,
+ ParseContext context, boolean includeMissingRows)
+ throws IOException, SAXException, TikaException {
- OpenDocumentBodyHandler dh = new OpenDocumentBodyHandler(handler,
context, metadata);
+ OpenDocumentBodyHandler dh =
+ new OpenDocumentBodyHandler(handler, context, metadata,
includeMissingRows);
tis.setCloseShield();
try {
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
index 9c4124b9fe..01b96cead9 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
@@ -78,6 +78,7 @@ public class OpenDocumentParser implements Parser {
*/
public static class Config {
public boolean extractMacros = false;
+ public boolean includeMissingRows = false;
}
private static final Set<MediaType> SUPPORTED_TYPES =
Collections.unmodifiableSet(
@@ -122,6 +123,7 @@ public class OpenDocumentParser implements Parser {
private Parser content = new OpenDocumentContentParser();
private boolean extractMacros = false;
+ private boolean includeMissingRows = false;
public OpenDocumentParser() {
}
@@ -133,6 +135,7 @@ public class OpenDocumentParser implements Parser {
*/
public OpenDocumentParser(Config config) {
this.extractMacros = config.extractMacros;
+ this.includeMissingRows = config.includeMissingRows;
}
/**
@@ -209,6 +212,19 @@ public class OpenDocumentParser implements Parser {
return extractMacros;
}
+ /**
+ * As in the Excel parsers: whether each empty spreadsheet row before a
row with
+ * content is written as an empty row; default false. Trailing empty rows
are always
+ * dropped, and at most 100,000 are written per document.
+ */
+ public void setIncludeMissingRows(boolean includeMissingRows) {
+ this.includeMissingRows = includeMissingRows;
+ }
+
+ public boolean isIncludeMissingRows() {
+ return includeMissingRows;
+ }
+
/** Re-opened from the zip on rewind rather than cached: the entry is in
the container already. */
private static TikaInputStream entryStream(ZipFile zipFile,
ZipArchiveEntry entry) {
return TikaInputStream.get(() -> zipFile.getInputStream(entry), new
TemporaryResources(), null);
@@ -300,14 +316,16 @@ public class OpenDocumentParser implements Parser {
meta.parse(tisZip, new DefaultHandler(), metadata, context);
} else if (entry.getName().endsWith("content.xml")) {
if (content instanceof OpenDocumentContentParser) {
- ((OpenDocumentContentParser)
content).parseInternal(tisZip, handler, metadata, context);
+ ((OpenDocumentContentParser)
content).parseInternal(tisZip, handler, metadata, context,
+ includeMissingRows);
} else {
// Foreign content parser was set:
content.parse(TikaInputStream.get(tisZip), handler, metadata,
context);
}
} else if (entry.getName().endsWith("styles.xml")) {
if (content instanceof OpenDocumentContentParser) {
- ((OpenDocumentContentParser)
content).parseInternal(tisZip, handler, metadata, context);
+ ((OpenDocumentContentParser)
content).parseInternal(tisZip, handler, metadata, context,
+ includeMissingRows);
} else {
// Foreign content parser was set:
content.parse(TikaInputStream.get(tisZip), handler, metadata,
context);
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
index 17b5ba6ec6..441c96616c 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
@@ -38,11 +38,17 @@ import org.apache.tika.sax.ContentHandlerDecorator;
* ODF stores runs of identical cells or rows once with a count. One with
content is
* replayed. Empty ones, repeated or not, are held back and written only if a
sibling
* with content follows before the parent closes, so the filler that pads a
sheet out to
- * its last column or row (often thousands of cells, a million rows) is
dropped.
+ * its last column or row (often thousands of cells, a million rows) is
dropped. Gaps
+ * before content are written up to {@code maxEmptyWritten} empty elements per
document:
+ * cells pad out to the next value's column, as the Excel parsers do, and
empty rows are
+ * dropped unless asked for.
+ * <p>
+ * Padding and every replayed copy are charged to a shared {@link Budget} at
one unit per
+ * character and per element (each element writes a tab or newline).
*/
class RepeatedElementHandler extends ContentHandlerDecorator {
- /** Characters a document may add by replaying, beyond each element's
first copy. */
+ /** Characters and elements a document may add by padding and replaying. */
static final long MAX_REPLAYED_CHARS = 10_000_000L;
/** Replay budget shared by the cell and row handlers of one document. */
@@ -61,13 +67,27 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
replayedChars += extra;
return true;
}
+ warn();
+ return false;
+ }
+
+ /** Takes as much of {@code extra} as fits; warns once when not all of
it does. */
+ long takeUpTo(long extra) {
+ long granted = Math.min(extra, MAX_REPLAYED_CHARS - replayedChars);
+ replayedChars += granted;
+ if (granted < extra) {
+ warn();
+ }
+ return granted;
+ }
+
+ private void warn() {
if (!warned && metadata != null) {
metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
"OpenDocument repeated cells/rows exceeded " +
MAX_REPLAYED_CHARS
+ " replayed characters; the rest were written
once");
}
warned = true;
- return false;
}
}
@@ -88,6 +108,8 @@ class RepeatedElementHandler extends ContentHandlerDecorator
{
}
}
+ private static final Attributes EMPTY_ATTRIBUTES = new AttributesImpl();
+
private interface Event {
void replay(ContentHandler h) throws SAXException;
}
@@ -96,6 +118,8 @@ class RepeatedElementHandler extends ContentHandlerDecorator
{
private final String element;
private final String parent;
private final int maxRepeat;
+ private long emptyLeft;
+ private final String emptyChild;
private final Budget budget;
private int nextRepeat;
@@ -107,20 +131,25 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
private int repeat;
private boolean hasContent;
private long chars;
+ private long elements;
private final Deque<EmptyRun> pendingEmpty = new ArrayDeque<>();
private long pendingCount;
/**
* @param element the XHTML element that repeats ({@code td} or {@code tr})
* @param parent the element whose end drops pending empty copies
+ * @param maxEmptyWritten most empty elements written before content, per
document
+ * @param emptyChild element written inside each empty one, or null
*/
RepeatedElementHandler(ContentHandler out, String element, String parent,
int maxRepeat,
- Budget budget) {
+ long maxEmptyWritten, String emptyChild, Budget
budget) {
super(out);
this.out = out;
this.element = element;
this.parent = parent;
this.maxRepeat = maxRepeat;
+ this.emptyLeft = maxEmptyWritten;
+ this.emptyChild = emptyChild;
this.budget = budget;
}
@@ -151,6 +180,7 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
throws SAXException {
if (buffer != null) {
depth++;
+ elements++;
if (localName.equals("img")) {
hasContent = true;
}
@@ -164,6 +194,7 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
repeat = Math.max(1, nextRepeat);
hasContent = false;
chars = 0;
+ elements = 1;
nextRepeat = 0;
AttributesImpl copy = new AttributesImpl(atts);
bufferUri = uri;
@@ -193,7 +224,7 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
}
writePendingEmpty();
int copies = repeat;
- if (copies > 1 && !budget.take(chars * (copies - 1))) {
+ if (copies > 1 && !budget.take((chars + elements) * (copies - 1)))
{
copies = 1;
}
for (int i = 0; i < copies; i++) {
@@ -241,7 +272,7 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
}
private void hold(String uri, String localName, String name, Attributes
atts, int count) {
- int n = (int) Math.min(count, maxRepeat - pendingCount);
+ int n = (int) Math.min(count, Math.min(maxRepeat, emptyLeft) -
pendingCount);
if (n <= 0) {
return;
}
@@ -260,9 +291,16 @@ class RepeatedElementHandler extends
ContentHandlerDecorator {
}
private void writePendingEmpty() throws SAXException {
+ int units = emptyChild == null ? 1 : 2;
+ long n = budget.takeUpTo(pendingCount * units) / units;
+ emptyLeft -= n;
for (EmptyRun run : pendingEmpty) {
- for (int i = 0; i < run.count; i++) {
+ for (int i = 0; i < run.count && n > 0; i++, n--) {
super.startElement(run.uri, run.localName, run.name, run.atts);
+ if (emptyChild != null) {
+ super.startElement(run.uri, emptyChild, emptyChild,
EMPTY_ATTRIBUTES);
+ super.endElement(run.uri, emptyChild, emptyChild);
+ }
super.endElement(run.uri, run.localName, run.name);
}
}
diff --git
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
index 5a0b87f7ef..cb554adc1a 100644
---
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
+++
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
@@ -21,9 +21,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.junit.jupiter.api.Test;
@@ -31,8 +35,11 @@ import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import org.apache.tika.TikaTest;
+import org.apache.tika.config.JsonConfig;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.ToXMLContentHandler;
@@ -119,7 +126,7 @@ public class OpenDocumentContentParserTest extends TikaTest
{
+ "<tr><tdcolspan=\"2\"><p>wide</p></td><td><p>c</p></td></tr>"
+
"<tr><td><p>r</p></td><td><p>s</p></td><td><p>s</p></td></tr>"
+
"<tr><td><p>r</p></td><td><p>s</p></td><td><p>s</p></td></tr>"
- + "<tr/><tr/><tr><td><p>z</p></td></tr>", table);
+ + "<tr><td><p>z</p></td></tr>", table);
}
@Test
@@ -170,4 +177,208 @@ public class OpenDocumentContentParserTest extends
TikaTest {
assertContains("first", out.toString());
assertContains("kept", out.toString());
}
+
+ private static final String NS =
+ "
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+ + "
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+ + " xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\"";
+
+ private static final String SHEET_MIME =
"application/vnd.oasis.opendocument.spreadsheet";
+
+ private static String row(String... values) {
+ StringBuilder sb = new StringBuilder("<table:table-row>");
+ for (String v : values) {
+
sb.append("<table:table-cell><text:p>").append(v).append("</text:p></table:table-cell>");
+ }
+ return sb.append("</table:table-row>").toString();
+ }
+
+ private static String emptyRows(int n) {
+ return "<table:table-row table:number-rows-repeated=\"" + n + "\">"
+ + "<table:table-cell
table:number-columns-repeated=\"16384\"/></table:table-row>";
+ }
+
+ private static String emptyCells(int n) {
+ return "<table:table-cell table:number-columns-repeated=\"" + n +
"\"/>";
+ }
+
+ private static byte[] fods(String rows) {
+ return ("<?xml version=\"1.0\" encoding=\"UTF-8\"?><office:document" +
NS
+ + " office:version=\"1.3\" office:mimetype=\"" + SHEET_MIME +
"\">"
+ + "<office:body><office:spreadsheet><table:table
table:name=\"S\">" + rows
+ +
"</table:table></office:spreadsheet></office:body></office:document>")
+ .getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static byte[] ods(String rows) throws Exception {
+ String content = "<?xml version=\"1.0\"
encoding=\"UTF-8\"?><office:document-content" + NS
+ + " office:version=\"1.3\"><office:body><office:spreadsheet>"
+ + "<table:table table:name=\"S\">" + rows
+ +
"</table:table></office:spreadsheet></office:body></office:document-content>";
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try (ZipOutputStream zos = new ZipOutputStream(bos)) {
+ ZipEntry mime = new ZipEntry("mimetype");
+ mime.setMethod(ZipEntry.STORED);
+ byte[] mimeBytes = SHEET_MIME.getBytes(StandardCharsets.US_ASCII);
+ CRC32 crc = new CRC32();
+ crc.update(mimeBytes);
+ mime.setSize(mimeBytes.length);
+ mime.setCrc(crc.getValue());
+ zos.putNextEntry(mime);
+ zos.write(mimeBytes);
+ zos.closeEntry();
+ zos.putNextEntry(new ZipEntry("content.xml"));
+ zos.write(content.getBytes(StandardCharsets.UTF_8));
+ zos.closeEntry();
+ }
+ return bos.toByteArray();
+ }
+
+ private XMLResult autoDetect(byte[] bytes, String name, Metadata metadata)
throws Exception {
+ return autoDetect(bytes, name, metadata, AUTO_DETECT_PARSER);
+ }
+
+ private XMLResult autoDetect(byte[] bytes, String name, Metadata metadata,
Parser parser)
+ throws Exception {
+ metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, name);
+ try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+ return getXML(tis, parser, metadata);
+ }
+ }
+
+ private static final String INCLUDE_MISSING_ROWS =
"{\"includeMissingRows\": true}";
+
+ // the AutoDetectParser wrapper keeps the zip-bomb guard in play
+ private static Parser odsWithMissingRows() {
+ return new AutoDetectParser(
+ new
OpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS)));
+ }
+
+ private static Parser fodsWithMissingRows() {
+ return new AutoDetectParser(
+ new
FlatOpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS)));
+ }
+
+ private static String table(String xml) {
+ return xml.substring(xml.indexOf("<table>"),
xml.indexOf("</table>")).replaceAll("\\s+", "");
+ }
+
+ private static int count(String haystack, String needle) {
+ return haystack.split(Pattern.quote(needle), -1).length - 1;
+ }
+
+ @Test
+ public void testMillionRowGapFods() throws Exception {
+ byte[] bytes = fods(row("before") + emptyRows(1048565) + row("after"));
+ String xml = autoDetect(bytes, "gap.fods", new Metadata()).xml;
+
assertEquals("<table><tr><td><p>before</p></td></tr><tr><td><p>after</p></td></tr>",
+ table(xml));
+ assertTrue(xml.length() < 2000, "output length " + xml.length());
+ }
+
+ @Test
+ public void testMillionRowGapOds() throws Exception {
+ byte[] bytes = ods(row("before") + emptyRows(1048565) + row("after"));
+ String xml = autoDetect(bytes, "gap.ods", new Metadata()).xml;
+
assertEquals("<table><tr><td><p>before</p></td></tr><tr><td><p>after</p></td></tr>",
+ table(xml));
+ }
+
+ @Test
+ public void testWideCellGapInRepeatedRows() throws Exception {
+ byte[] bytes = fods("<table:table-row
table:number-rows-repeated=\"100000\">"
+ + emptyCells(16383) +
"<table:table-cell><text:p>x</text:p></table:table-cell>"
+ + "</table:table-row>" + row("after"));
+ Metadata metadata = new Metadata();
+ String xml = autoDetect(bytes, "wide.fods", metadata).xml;
+ // the row is too costly to repeat, so it is written once
+ assertEquals(16383, count(xml, "<td />"));
+ assertEquals(1, count(xml, "<p>x</p>"));
+ assertContains("<p>after</p>", xml);
+ assertContains("written once",
+ metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+ }
+
+ @Test
+ public void testSmallGapsKept() throws Exception {
+ byte[] bytes =
fods("<table:table-row><table:table-cell><text:p>a</text:p></table:table-cell>"
+ + emptyCells(3) +
"<table:table-cell><text:p>b</text:p></table:table-cell>"
+ + emptyCells(16379) + "</table:table-row>" + emptyRows(5) +
row("c")
+ + emptyRows(1048000));
+ Metadata metadata = new Metadata();
+ String xml = autoDetect(bytes, "small.fods", metadata).xml;
+
assertEquals("<table><tr><td><p>a</p></td><td/><td/><td/><td><p>b</p></td></tr>"
+ + "<tr><td><p>c</p></td></tr>", table(xml));
+ assertEquals(null,
metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+ }
+
+ @Test
+ public void testRepeatedValuesExpanded() throws Exception {
+ byte[] bytes = fods("<table:table-row
table:number-rows-repeated=\"1000\">"
+ + "<table:table-cell
table:number-columns-repeated=\"3\"><text:p>v</text:p>"
+ + "</table:table-cell></table:table-row>");
+ String xml = autoDetect(bytes, "values.fods", new Metadata()).xml;
+ assertEquals(3000, count(xml, "<p>v</p>"));
+ }
+
+ @Test
+ public void testEmptyCellBudgetExhausted() throws Exception {
+ // each row charges 1024 for its gap; the budget runs out after 9765
rows
+ StringBuilder rows = new StringBuilder();
+ int n = 10000;
+ for (int i = 0; i < n; i++) {
+ rows.append("<table:table-row>").append(emptyCells(1024))
+ .append("<table:table-cell><text:p>r").append(i)
+ .append("</text:p></table:table-cell></table:table-row>");
+ }
+ Metadata metadata = new Metadata();
+ String xml;
+ try (TikaInputStream tis = TikaInputStream.get(fods(rows.toString())))
{
+ xml = getXML(tis, new FlatOpenDocumentParser(), metadata).xml;
+ }
+ String first = xml.substring(xml.indexOf("<tr>"),
xml.indexOf("</tr>"));
+ assertEquals(1024, count(first, "<td />"));
+ String last = xml.substring(xml.lastIndexOf("<tr>"),
xml.lastIndexOf("</tr>"));
+ assertEquals(0, count(last, "<td />"));
+ assertContains("<p>r" + (n - 1) + "</p>", last);
+ assertContains("written once",
+ metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+ }
+
+ @Test
+ public void testIncludeMissingRowsConfig() throws Exception {
+ assertFalse(new OpenDocumentParser().isIncludeMissingRows());
+ assertFalse(new FlatOpenDocumentParser().isIncludeMissingRows());
+ assertTrue(new
OpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS))
+ .isIncludeMissingRows());
+ assertTrue(new
FlatOpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS))
+ .isIncludeMissingRows());
+ }
+
+ @Test
+ public void testIncludeMissingRows() throws Exception {
+ String rows = row("a") +
"<table:table-row><table:table-cell/></table:table-row>"
+ + emptyRows(2) + row("b") + emptyRows(1048000);
+ String expected =
"<table><tr><td><p>a</p></td></tr><tr><td/></tr><tr><td/></tr><tr><td/></tr>"
+ + "<tr><td><p>b</p></td></tr>";
+ assertEquals(expected,
+ table(autoDetect(fods(rows), "e.fods", new Metadata(),
fodsWithMissingRows()).xml));
+ assertEquals(expected,
+ table(autoDetect(ods(rows), "e.ods", new Metadata(),
odsWithMissingRows()).xml));
+
assertEquals("<table><tr><td><p>a</p></td></tr><tr><td><p>b</p></td></tr>",
+ table(autoDetect(fods(rows), "e.fods", new Metadata()).xml));
+ }
+
+ @Test
+ public void testIncludeMissingRowsMillionRowGap() throws Exception {
+ String rows = row("before") + emptyRows(1048565) + row("after");
+ for (boolean packaged : new boolean[]{false, true}) {
+ String xml = packaged
+ ? autoDetect(ods(rows), "gap.ods", new Metadata(),
odsWithMissingRows()).xml
+ : autoDetect(fods(rows), "gap.fods", new Metadata(),
fodsWithMissingRows()).xml;
+ assertEquals(100_000, xml.split("<tr>\\s*<td />\\s*</tr>",
-1).length - 1);
+ assertTrue(xml.length() < 3_000_000, "output length " +
xml.length());
+ assertContains("<p>after</p>", xml);
+ }
+ }
}