This is an automated email from the ASF dual-hosted git repository.

tballison pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tika.git


The following commit(s) were added to refs/heads/main by this push:
     new ed8a8913a2 TIKA-5014: drop empty ODS rows, pad cells only before 
content, charge all writes (#3600)
ed8a8913a2 is described below

commit ed8a8913a232395cd764fb6cafae3123f72a193e
Author: Tim Allison <[email protected]>
AuthorDate: Thu Nov 12 14:03:33 2026 -0500

    TIKA-5014: drop empty ODS rows, pad cells only before content, charge all 
writes (#3600)
---
 CHANGES.txt                                        |  14 +-
 docs/modules/ROOT/pages/formats.adoc               |  18 ++
 .../org/apache/tika/parser/odf/ODFParserTest.java  |  28 +++
 .../configs/tika-config-ods-missing-rows.json      |  17 ++
 .../tika/parser/odf/FlatOpenDocumentParser.java    |  20 +-
 .../tika/parser/odf/OpenDocumentBodyHandler.java   |  12 +-
 .../tika/parser/odf/OpenDocumentContentParser.java |   9 +-
 .../apache/tika/parser/odf/OpenDocumentParser.java |  22 ++-
 .../tika/parser/odf/RepeatedElementHandler.java    |  52 ++++-
 .../parser/odf/OpenDocumentContentParserTest.java  | 213 ++++++++++++++++++++-
 10 files changed, 385 insertions(+), 20 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index 0e94fe707e..e524a8437b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -385,10 +385,16 @@ Release 4.3.0 - ???
 
    * The OpenDocument parser writes repeated cells and rows 
(table:number-columns-repeated,
      table:number-rows-repeated) once per repeat; a cell was written once with 
that
-     colspan and a row once, so values repeated across columns or rows were 
lost. Empty
-     cells and rows at the end of a row or table, such as the filler out to a 
sheet's
-     last column and row, are dropped. Past 10 million characters of repeated 
content
-     per document the rest are written once, with a tk:exception:warn entry 
(TIKA-5014).
+     colspan and a row once, so values repeated across columns or rows were 
lost. As in
+     the Excel parsers, empty rows are dropped and empty cells are written 
only to pad
+     out to the next value in their row. Past 10 million characters and 
elements of
+     padding and repeated content per document, the rest are written once and 
padding
+     is dropped, with a tk:exception:warn entry (TIKA-5014).
+
+   * The OpenDocument parsers (open-document-parser, 
flat-open-document-parser) take
+     includeMissingRows, as the Excel parsers do: when true, each empty 
spreadsheet row
+     before a row with content is written as an empty row, up to 100,000 per 
document.
+     The default is false (TIKA-5014).
 
    * VLM text recognizers keep superscripts and subscripts: inline <sup>/<sub> 
in model
      output (x<sup>2</sup>, H<sub>2</sub>O, footnote markers) reach the XHTML 
as sup/sub
diff --git a/docs/modules/ROOT/pages/formats.adoc 
b/docs/modules/ROOT/pages/formats.adoc
index fd3f3af91a..0390ff7b9e 100644
--- a/docs/modules/ROOT/pages/formats.adoc
+++ b/docs/modules/ROOT/pages/formats.adoc
@@ -85,6 +85,24 @@ ODF is based;
 
link:{api}/org/apache/tika/parser/odf/FlatOpenDocumentParser.html[FlatOpenDocumentParser]
 handles the single-file flat XML variants (`.fodt`, `.fods`, `.fodp`).
 
+Both write spreadsheet tables as the Excel parsers do: empty rows are dropped,
+empty cells are written only to pad out to the next value in their row, and
+the filler at the end of a row or sheet is dropped. Set `includeMissingRows`
+to write each empty row before a row with content, as `includeMissingRows`
+does for Excel; at most 100,000 are written per document. Set `extractMacros`
+to extract Basic macros.
+
+[source,json]
+----
+{
+  "parsers": [
+    { "default-parser": {} },
+    { "open-document-parser": { "includeMissingRows": true } },
+    { "flat-open-document-parser": { "includeMissingRows": true } }
+  ]
+}
+----
+
 == iWork document formats
 
 The Apple iWork formats (Pages, Numbers, Keynote) are supported by
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
index c2b8f82842..92391130b2 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/java/org/apache/tika/parser/odf/ODFParserTest.java
@@ -18,6 +18,7 @@ package org.apache.tika.parser.odf;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 import org.junit.jupiter.api.BeforeAll;
@@ -25,6 +26,7 @@ import org.junit.jupiter.api.Test;
 
 import org.apache.tika.TikaLoaderHelper;
 import org.apache.tika.TikaTest;
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.HttpHeaders;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.PagedText;
@@ -211,4 +213,30 @@ public class ODFParserTest extends TikaTest {
     private void assertImageContentType(String expected, String actual) {
         assertEquals(expected, actual);
     }
+
+    @Test
+    public void testIncludeMissingRowsConfig() throws Exception {
+        Parser parser = 
TikaLoaderHelper.getLoader("tika-config-ods-missing-rows.json")
+                .loadAutoDetectParser();
+        String fods = "<?xml version=\"1.0\" 
encoding=\"UTF-8\"?><office:document"
+                + " 
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+                + " 
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+                + " 
xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\""
+                + " 
office:mimetype=\"application/vnd.oasis.opendocument.spreadsheet\">"
+                + "<office:body><office:spreadsheet><table:table 
table:name=\"S\">"
+                + 
"<table:table-row><table:table-cell><text:p>a</text:p></table:table-cell>"
+                + "</table:table-row><table:table-row 
table:number-rows-repeated=\"3\">"
+                + "<table:table-cell/></table:table-row>"
+                + 
"<table:table-row><table:table-cell><text:p>b</text:p></table:table-cell>"
+                + 
"</table:table-row></table:table></office:spreadsheet></office:body>"
+                + "</office:document>";
+        Metadata metadata = new Metadata();
+        metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, "rows.fods");
+        String xml;
+        try (TikaInputStream tis = 
TikaInputStream.get(fods.getBytes(StandardCharsets.UTF_8))) {
+            xml = getXML(tis, parser, metadata).xml;
+        }
+        assertEquals(3, xml.split("<tr>\\s*<td />\\s*</tr>", -1).length - 1);
+        assertContains("<p>b</p>", xml);
+    }
 }
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/configs/tika-config-ods-missing-rows.json
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/configs/tika-config-ods-missing-rows.json
new file mode 100644
index 0000000000..ae68a2c7de
--- /dev/null
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-integration-tests/src/test/resources/configs/tika-config-ods-missing-rows.json
@@ -0,0 +1,17 @@
+{
+  "parsers": [
+    {
+      "default-parser" : {}
+    },
+    {
+      "open-document-parser": {
+        "includeMissingRows": true
+      }
+    },
+    {
+      "flat-open-document-parser": {
+        "includeMissingRows": true
+      }
+    }
+  ]
+}
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
index 26a31af9b5..cd665101b3 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/FlatOpenDocumentParser.java
@@ -51,6 +51,7 @@ public class FlatOpenDocumentParser implements Parser {
      */
     public static class Config {
         public boolean extractMacros = false;
+        public boolean includeMissingRows = false;
     }
 
     static final MediaType FLAT_OD =
@@ -68,6 +69,7 @@ public class FlatOpenDocumentParser implements Parser {
             .unmodifiableSet(new HashSet<>(Arrays.asList(FLAT_OD, FLAT_ODT, 
FLAT_ODP, FLAT_ODS)));
 
     private boolean extractMacros = false;
+    private boolean includeMissingRows = false;
 
     public FlatOpenDocumentParser() {
     }
@@ -79,6 +81,7 @@ public class FlatOpenDocumentParser implements Parser {
      */
     public FlatOpenDocumentParser(Config config) {
         this.extractMacros = config.extractMacros;
+        this.includeMissingRows = config.includeMissingRows;
     }
 
     /**
@@ -144,10 +147,20 @@ public class FlatOpenDocumentParser implements Parser {
     public boolean isExtractMacros() {
         return extractMacros;
     }
+
+    /** See {@link OpenDocumentParser#setIncludeMissingRows(boolean)}. */
+    public void setIncludeMissingRows(boolean includeMissingRows) {
+        this.includeMissingRows = includeMissingRows;
+    }
+
+    public boolean isIncludeMissingRows() {
+        return includeMissingRows;
+    }
     private FlatOpenDocumentParserHandler getContentHandler(ContentHandler 
handler,
                                                             Metadata metadata,
                                                             ParseContext 
context) {
-        return new FlatOpenDocumentParserHandler(handler, metadata, context, 
extractMacros);
+        return new FlatOpenDocumentParserHandler(handler, metadata, context, 
extractMacros,
+                includeMissingRows);
     }
 
     private static class FlatOpenDocumentParserHandler extends 
ContentHandlerDecorator {
@@ -181,11 +194,12 @@ public class FlatOpenDocumentParser implements Parser {
         private int outerSectionDepth = 0;
 
         private FlatOpenDocumentParserHandler(ContentHandler baseHandler, 
Metadata metadata,
-                                              ParseContext parseContext, 
boolean extractMacros) {
+                                              ParseContext parseContext, 
boolean extractMacros,
+                                              boolean includeMissingRows) {
             this.extractMacros = extractMacros;
 
             this.bodyHandler = new OpenDocumentBodyHandler(new 
NSNormalizerContentHandler(baseHandler),
-                            parseContext, metadata);
+                            parseContext, metadata, includeMissingRows);
 
             this.metadataHandler = new NSNormalizerContentHandler(
                     OpenDocumentMetaParser.getContentHandler(metadata, 
parseContext));
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
index 3435dbfd2a..e1fa87fadc 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentBodyHandler.java
@@ -75,6 +75,8 @@ class OpenDocumentBodyHandler extends 
ElementMappingContentHandler {
     // ODF 1.2+ sheet limits
     private static final int MAX_COLUMNS = 16384;
     private static final int MAX_ROWS = 1048576;
+    // with includeMissingRows: keeps a 1M-row gap below the zip-bomb guard's 
1M-char threshold
+    private static final int MAX_EMPTY_ROWS_WRITTEN = 100_000;
     private static final Attributes ANNOTATION_ATTRIBUTES = 
buildAttributes(CLASS, "annotation");
     private static final Attributes NOTE_ATTRIBUTES = buildAttributes(CLASS, 
"note");
     private static final Attributes NOTES_ATTRIBUTES = buildAttributes(CLASS, 
"notes");
@@ -154,13 +156,19 @@ class OpenDocumentBodyHandler extends 
ElementMappingContentHandler {
     // their formatting in the captured XHTML.
     private int anchorDepth = 0;
     OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext, 
Metadata metadata) {
+        this(handler, parseContext, metadata, false);
+    }
+
+    OpenDocumentBodyHandler(ContentHandler handler, ParseContext parseContext, 
Metadata metadata,
+                            boolean includeMissingRows) {
         this(new RepeatedElementHandler(handler, "tr", "table", MAX_ROWS,
+                includeMissingRows ? MAX_EMPTY_ROWS_WRITTEN : 0, "td",
                 new RepeatedElementHandler.Budget(metadata)), parseContext);
     }
 
     private OpenDocumentBodyHandler(RepeatedElementHandler rows, ParseContext 
parseContext) {
-        this(new RepeatedElementHandler(rows, "td", "tr", MAX_COLUMNS, 
rows.budget()), rows,
-                parseContext);
+        this(new RepeatedElementHandler(rows, "td", "tr", MAX_COLUMNS, 
Long.MAX_VALUE, null,
+                rows.budget()), rows, parseContext);
     }
 
     private OpenDocumentBodyHandler(RepeatedElementHandler cells, 
RepeatedElementHandler rows,
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
index 8fce0ef133..63028c4031 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentContentParser.java
@@ -48,8 +48,15 @@ public class OpenDocumentContentParser implements Parser {
 
     void parseInternal(TikaInputStream tis, final ContentHandler handler, 
Metadata metadata,
                        ParseContext context) throws IOException, SAXException, 
TikaException {
+        parseInternal(tis, handler, metadata, context, false);
+    }
+
+    void parseInternal(TikaInputStream tis, final ContentHandler handler, 
Metadata metadata,
+                       ParseContext context, boolean includeMissingRows)
+            throws IOException, SAXException, TikaException {
 
-        OpenDocumentBodyHandler dh = new OpenDocumentBodyHandler(handler, 
context, metadata);
+        OpenDocumentBodyHandler dh =
+                new OpenDocumentBodyHandler(handler, context, metadata, 
includeMissingRows);
 
         tis.setCloseShield();
         try {
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
index 9c4124b9fe..01b96cead9 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/OpenDocumentParser.java
@@ -78,6 +78,7 @@ public class OpenDocumentParser implements Parser {
      */
     public static class Config {
         public boolean extractMacros = false;
+        public boolean includeMissingRows = false;
     }
 
     private static final Set<MediaType> SUPPORTED_TYPES = 
Collections.unmodifiableSet(
@@ -122,6 +123,7 @@ public class OpenDocumentParser implements Parser {
 
     private Parser content = new OpenDocumentContentParser();
     private boolean extractMacros = false;
+    private boolean includeMissingRows = false;
 
     public OpenDocumentParser() {
     }
@@ -133,6 +135,7 @@ public class OpenDocumentParser implements Parser {
      */
     public OpenDocumentParser(Config config) {
         this.extractMacros = config.extractMacros;
+        this.includeMissingRows = config.includeMissingRows;
     }
 
     /**
@@ -209,6 +212,19 @@ public class OpenDocumentParser implements Parser {
         return extractMacros;
     }
 
+    /**
+     * As in the Excel parsers: whether each empty spreadsheet row before a 
row with
+     * content is written as an empty row; default false. Trailing empty rows 
are always
+     * dropped, and at most 100,000 are written per document.
+     */
+    public void setIncludeMissingRows(boolean includeMissingRows) {
+        this.includeMissingRows = includeMissingRows;
+    }
+
+    public boolean isIncludeMissingRows() {
+        return includeMissingRows;
+    }
+
     /** Re-opened from the zip on rewind rather than cached: the entry is in 
the container already. */
     private static TikaInputStream entryStream(ZipFile zipFile, 
ZipArchiveEntry entry) {
         return TikaInputStream.get(() -> zipFile.getInputStream(entry), new 
TemporaryResources(), null);
@@ -300,14 +316,16 @@ public class OpenDocumentParser implements Parser {
                 meta.parse(tisZip, new DefaultHandler(), metadata, context);
         } else if (entry.getName().endsWith("content.xml")) {
             if (content instanceof OpenDocumentContentParser) {
-                    ((OpenDocumentContentParser) 
content).parseInternal(tisZip, handler, metadata, context);
+                    ((OpenDocumentContentParser) 
content).parseInternal(tisZip, handler, metadata, context,
+                            includeMissingRows);
             } else {
                 // Foreign content parser was set:
                 content.parse(TikaInputStream.get(tisZip), handler, metadata, 
context);
             }
         } else if (entry.getName().endsWith("styles.xml")) {
             if (content instanceof OpenDocumentContentParser) {
-                    ((OpenDocumentContentParser) 
content).parseInternal(tisZip, handler, metadata, context);
+                    ((OpenDocumentContentParser) 
content).parseInternal(tisZip, handler, metadata, context,
+                            includeMissingRows);
             } else {
                 // Foreign content parser was set:
                 content.parse(TikaInputStream.get(tisZip), handler, metadata, 
context);
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
index 17b5ba6ec6..441c96616c 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/main/java/org/apache/tika/parser/odf/RepeatedElementHandler.java
@@ -38,11 +38,17 @@ import org.apache.tika.sax.ContentHandlerDecorator;
  * ODF stores runs of identical cells or rows once with a count. One with 
content is
  * replayed. Empty ones, repeated or not, are held back and written only if a 
sibling
  * with content follows before the parent closes, so the filler that pads a 
sheet out to
- * its last column or row (often thousands of cells, a million rows) is 
dropped.
+ * its last column or row (often thousands of cells, a million rows) is 
dropped. Gaps
+ * before content are written up to {@code maxEmptyWritten} empty elements per 
document:
+ * cells pad out to the next value's column, as the Excel parsers do, and 
empty rows are
+ * dropped unless asked for.
+ * <p>
+ * Padding and every replayed copy are charged to a shared {@link Budget} at 
one unit per
+ * character and per element (each element writes a tab or newline).
  */
 class RepeatedElementHandler extends ContentHandlerDecorator {
 
-    /** Characters a document may add by replaying, beyond each element's 
first copy. */
+    /** Characters and elements a document may add by padding and replaying. */
     static final long MAX_REPLAYED_CHARS = 10_000_000L;
 
     /** Replay budget shared by the cell and row handlers of one document. */
@@ -61,13 +67,27 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
                 replayedChars += extra;
                 return true;
             }
+            warn();
+            return false;
+        }
+
+        /** Takes as much of {@code extra} as fits; warns once when not all of 
it does. */
+        long takeUpTo(long extra) {
+            long granted = Math.min(extra, MAX_REPLAYED_CHARS - replayedChars);
+            replayedChars += granted;
+            if (granted < extra) {
+                warn();
+            }
+            return granted;
+        }
+
+        private void warn() {
             if (!warned && metadata != null) {
                 metadata.add(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING,
                         "OpenDocument repeated cells/rows exceeded " + 
MAX_REPLAYED_CHARS
                                 + " replayed characters; the rest were written 
once");
             }
             warned = true;
-            return false;
         }
     }
 
@@ -88,6 +108,8 @@ class RepeatedElementHandler extends ContentHandlerDecorator 
{
         }
     }
 
+    private static final Attributes EMPTY_ATTRIBUTES = new AttributesImpl();
+
     private interface Event {
         void replay(ContentHandler h) throws SAXException;
     }
@@ -96,6 +118,8 @@ class RepeatedElementHandler extends ContentHandlerDecorator 
{
     private final String element;
     private final String parent;
     private final int maxRepeat;
+    private long emptyLeft;
+    private final String emptyChild;
     private final Budget budget;
 
     private int nextRepeat;
@@ -107,20 +131,25 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
     private int repeat;
     private boolean hasContent;
     private long chars;
+    private long elements;
     private final Deque<EmptyRun> pendingEmpty = new ArrayDeque<>();
     private long pendingCount;
 
     /**
      * @param element the XHTML element that repeats ({@code td} or {@code tr})
      * @param parent  the element whose end drops pending empty copies
+     * @param maxEmptyWritten most empty elements written before content, per 
document
+     * @param emptyChild      element written inside each empty one, or null
      */
     RepeatedElementHandler(ContentHandler out, String element, String parent, 
int maxRepeat,
-                           Budget budget) {
+                           long maxEmptyWritten, String emptyChild, Budget 
budget) {
         super(out);
         this.out = out;
         this.element = element;
         this.parent = parent;
         this.maxRepeat = maxRepeat;
+        this.emptyLeft = maxEmptyWritten;
+        this.emptyChild = emptyChild;
         this.budget = budget;
     }
 
@@ -151,6 +180,7 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
             throws SAXException {
         if (buffer != null) {
             depth++;
+            elements++;
             if (localName.equals("img")) {
                 hasContent = true;
             }
@@ -164,6 +194,7 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
             repeat = Math.max(1, nextRepeat);
             hasContent = false;
             chars = 0;
+            elements = 1;
             nextRepeat = 0;
             AttributesImpl copy = new AttributesImpl(atts);
             bufferUri = uri;
@@ -193,7 +224,7 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
             }
             writePendingEmpty();
             int copies = repeat;
-            if (copies > 1 && !budget.take(chars * (copies - 1))) {
+            if (copies > 1 && !budget.take((chars + elements) * (copies - 1))) 
{
                 copies = 1;
             }
             for (int i = 0; i < copies; i++) {
@@ -241,7 +272,7 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
     }
 
     private void hold(String uri, String localName, String name, Attributes 
atts, int count) {
-        int n = (int) Math.min(count, maxRepeat - pendingCount);
+        int n = (int) Math.min(count, Math.min(maxRepeat, emptyLeft) - 
pendingCount);
         if (n <= 0) {
             return;
         }
@@ -260,9 +291,16 @@ class RepeatedElementHandler extends 
ContentHandlerDecorator {
     }
 
     private void writePendingEmpty() throws SAXException {
+        int units = emptyChild == null ? 1 : 2;
+        long n = budget.takeUpTo(pendingCount * units) / units;
+        emptyLeft -= n;
         for (EmptyRun run : pendingEmpty) {
-            for (int i = 0; i < run.count; i++) {
+            for (int i = 0; i < run.count && n > 0; i++, n--) {
                 super.startElement(run.uri, run.localName, run.name, run.atts);
+                if (emptyChild != null) {
+                    super.startElement(run.uri, emptyChild, emptyChild, 
EMPTY_ATTRIBUTES);
+                    super.endElement(run.uri, emptyChild, emptyChild);
+                }
                 super.endElement(run.uri, run.localName, run.name);
             }
         }
diff --git 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
index 5a0b87f7ef..cb554adc1a 100644
--- 
a/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
+++ 
b/tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-miscoffice-module/src/test/java/org/apache/tika/parser/odf/OpenDocumentContentParserTest.java
@@ -21,9 +21,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.junit.jupiter.api.Test;
@@ -31,8 +35,11 @@ import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
 
 import org.apache.tika.TikaTest;
+import org.apache.tika.config.JsonConfig;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
 import org.apache.tika.sax.ToXMLContentHandler;
@@ -119,7 +126,7 @@ public class OpenDocumentContentParserTest extends TikaTest 
{
                 + "<tr><tdcolspan=\"2\"><p>wide</p></td><td><p>c</p></td></tr>"
                 + 
"<tr><td><p>r</p></td><td><p>s</p></td><td><p>s</p></td></tr>"
                 + 
"<tr><td><p>r</p></td><td><p>s</p></td><td><p>s</p></td></tr>"
-                + "<tr/><tr/><tr><td><p>z</p></td></tr>", table);
+                + "<tr><td><p>z</p></td></tr>", table);
     }
 
     @Test
@@ -170,4 +177,208 @@ public class OpenDocumentContentParserTest extends 
TikaTest {
         assertContains("first", out.toString());
         assertContains("kept", out.toString());
     }
+
+    private static final String NS =
+            " 
xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
+            + " 
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
+            + " xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\"";
+
+    private static final String SHEET_MIME = 
"application/vnd.oasis.opendocument.spreadsheet";
+
+    private static String row(String... values) {
+        StringBuilder sb = new StringBuilder("<table:table-row>");
+        for (String v : values) {
+            
sb.append("<table:table-cell><text:p>").append(v).append("</text:p></table:table-cell>");
+        }
+        return sb.append("</table:table-row>").toString();
+    }
+
+    private static String emptyRows(int n) {
+        return "<table:table-row table:number-rows-repeated=\"" + n + "\">"
+                + "<table:table-cell 
table:number-columns-repeated=\"16384\"/></table:table-row>";
+    }
+
+    private static String emptyCells(int n) {
+        return "<table:table-cell table:number-columns-repeated=\"" + n + 
"\"/>";
+    }
+
+    private static byte[] fods(String rows) {
+        return ("<?xml version=\"1.0\" encoding=\"UTF-8\"?><office:document" + 
NS
+                + " office:version=\"1.3\" office:mimetype=\"" + SHEET_MIME + 
"\">"
+                + "<office:body><office:spreadsheet><table:table 
table:name=\"S\">" + rows
+                + 
"</table:table></office:spreadsheet></office:body></office:document>")
+                .getBytes(StandardCharsets.UTF_8);
+    }
+
+    private static byte[] ods(String rows) throws Exception {
+        String content = "<?xml version=\"1.0\" 
encoding=\"UTF-8\"?><office:document-content" + NS
+                + " office:version=\"1.3\"><office:body><office:spreadsheet>"
+                + "<table:table table:name=\"S\">" + rows
+                + 
"</table:table></office:spreadsheet></office:body></office:document-content>";
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        try (ZipOutputStream zos = new ZipOutputStream(bos)) {
+            ZipEntry mime = new ZipEntry("mimetype");
+            mime.setMethod(ZipEntry.STORED);
+            byte[] mimeBytes = SHEET_MIME.getBytes(StandardCharsets.US_ASCII);
+            CRC32 crc = new CRC32();
+            crc.update(mimeBytes);
+            mime.setSize(mimeBytes.length);
+            mime.setCrc(crc.getValue());
+            zos.putNextEntry(mime);
+            zos.write(mimeBytes);
+            zos.closeEntry();
+            zos.putNextEntry(new ZipEntry("content.xml"));
+            zos.write(content.getBytes(StandardCharsets.UTF_8));
+            zos.closeEntry();
+        }
+        return bos.toByteArray();
+    }
+
+    private XMLResult autoDetect(byte[] bytes, String name, Metadata metadata) 
throws Exception {
+        return autoDetect(bytes, name, metadata, AUTO_DETECT_PARSER);
+    }
+
+    private XMLResult autoDetect(byte[] bytes, String name, Metadata metadata, 
Parser parser)
+            throws Exception {
+        metadata.set(TikaCoreProperties.RESOURCE_NAME_KEY, name);
+        try (TikaInputStream tis = TikaInputStream.get(bytes)) {
+            return getXML(tis, parser, metadata);
+        }
+    }
+
+    private static final String INCLUDE_MISSING_ROWS = 
"{\"includeMissingRows\": true}";
+
+    // the AutoDetectParser wrapper keeps the zip-bomb guard in play
+    private static Parser odsWithMissingRows() {
+        return new AutoDetectParser(
+                new 
OpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS)));
+    }
+
+    private static Parser fodsWithMissingRows() {
+        return new AutoDetectParser(
+                new 
FlatOpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS)));
+    }
+
+    private static String table(String xml) {
+        return xml.substring(xml.indexOf("<table>"), 
xml.indexOf("</table>")).replaceAll("\\s+", "");
+    }
+
+    private static int count(String haystack, String needle) {
+        return haystack.split(Pattern.quote(needle), -1).length - 1;
+    }
+
+    @Test
+    public void testMillionRowGapFods() throws Exception {
+        byte[] bytes = fods(row("before") + emptyRows(1048565) + row("after"));
+        String xml = autoDetect(bytes, "gap.fods", new Metadata()).xml;
+        
assertEquals("<table><tr><td><p>before</p></td></tr><tr><td><p>after</p></td></tr>",
+                table(xml));
+        assertTrue(xml.length() < 2000, "output length " + xml.length());
+    }
+
+    @Test
+    public void testMillionRowGapOds() throws Exception {
+        byte[] bytes = ods(row("before") + emptyRows(1048565) + row("after"));
+        String xml = autoDetect(bytes, "gap.ods", new Metadata()).xml;
+        
assertEquals("<table><tr><td><p>before</p></td></tr><tr><td><p>after</p></td></tr>",
+                table(xml));
+    }
+
+    @Test
+    public void testWideCellGapInRepeatedRows() throws Exception {
+        byte[] bytes = fods("<table:table-row 
table:number-rows-repeated=\"100000\">"
+                + emptyCells(16383) + 
"<table:table-cell><text:p>x</text:p></table:table-cell>"
+                + "</table:table-row>" + row("after"));
+        Metadata metadata = new Metadata();
+        String xml = autoDetect(bytes, "wide.fods", metadata).xml;
+        // the row is too costly to repeat, so it is written once
+        assertEquals(16383, count(xml, "<td />"));
+        assertEquals(1, count(xml, "<p>x</p>"));
+        assertContains("<p>after</p>", xml);
+        assertContains("written once",
+                metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+    }
+
+    @Test
+    public void testSmallGapsKept() throws Exception {
+        byte[] bytes = 
fods("<table:table-row><table:table-cell><text:p>a</text:p></table:table-cell>"
+                + emptyCells(3) + 
"<table:table-cell><text:p>b</text:p></table:table-cell>"
+                + emptyCells(16379) + "</table:table-row>" + emptyRows(5) + 
row("c")
+                + emptyRows(1048000));
+        Metadata metadata = new Metadata();
+        String xml = autoDetect(bytes, "small.fods", metadata).xml;
+        
assertEquals("<table><tr><td><p>a</p></td><td/><td/><td/><td><p>b</p></td></tr>"
+                + "<tr><td><p>c</p></td></tr>", table(xml));
+        assertEquals(null, 
metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+    }
+
+    @Test
+    public void testRepeatedValuesExpanded() throws Exception {
+        byte[] bytes = fods("<table:table-row 
table:number-rows-repeated=\"1000\">"
+                + "<table:table-cell 
table:number-columns-repeated=\"3\"><text:p>v</text:p>"
+                + "</table:table-cell></table:table-row>");
+        String xml = autoDetect(bytes, "values.fods", new Metadata()).xml;
+        assertEquals(3000, count(xml, "<p>v</p>"));
+    }
+
+    @Test
+    public void testEmptyCellBudgetExhausted() throws Exception {
+        // each row charges 1024 for its gap; the budget runs out after 9765 
rows
+        StringBuilder rows = new StringBuilder();
+        int n = 10000;
+        for (int i = 0; i < n; i++) {
+            rows.append("<table:table-row>").append(emptyCells(1024))
+                    .append("<table:table-cell><text:p>r").append(i)
+                    .append("</text:p></table:table-cell></table:table-row>");
+        }
+        Metadata metadata = new Metadata();
+        String xml;
+        try (TikaInputStream tis = TikaInputStream.get(fods(rows.toString()))) 
{
+            xml = getXML(tis, new FlatOpenDocumentParser(), metadata).xml;
+        }
+        String first = xml.substring(xml.indexOf("<tr>"), 
xml.indexOf("</tr>"));
+        assertEquals(1024, count(first, "<td />"));
+        String last = xml.substring(xml.lastIndexOf("<tr>"), 
xml.lastIndexOf("</tr>"));
+        assertEquals(0, count(last, "<td />"));
+        assertContains("<p>r" + (n - 1) + "</p>", last);
+        assertContains("written once",
+                metadata.get(TikaCoreProperties.TIKA_META_EXCEPTION_WARNING));
+    }
+
+    @Test
+    public void testIncludeMissingRowsConfig() throws Exception {
+        assertFalse(new OpenDocumentParser().isIncludeMissingRows());
+        assertFalse(new FlatOpenDocumentParser().isIncludeMissingRows());
+        assertTrue(new 
OpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS))
+                .isIncludeMissingRows());
+        assertTrue(new 
FlatOpenDocumentParser(JsonConfig.trusted(INCLUDE_MISSING_ROWS))
+                .isIncludeMissingRows());
+    }
+
+    @Test
+    public void testIncludeMissingRows() throws Exception {
+        String rows = row("a") + 
"<table:table-row><table:table-cell/></table:table-row>"
+                + emptyRows(2) + row("b") + emptyRows(1048000);
+        String expected = 
"<table><tr><td><p>a</p></td></tr><tr><td/></tr><tr><td/></tr><tr><td/></tr>"
+                + "<tr><td><p>b</p></td></tr>";
+        assertEquals(expected,
+                table(autoDetect(fods(rows), "e.fods", new Metadata(), 
fodsWithMissingRows()).xml));
+        assertEquals(expected,
+                table(autoDetect(ods(rows), "e.ods", new Metadata(), 
odsWithMissingRows()).xml));
+        
assertEquals("<table><tr><td><p>a</p></td></tr><tr><td><p>b</p></td></tr>",
+                table(autoDetect(fods(rows), "e.fods", new Metadata()).xml));
+    }
+
+    @Test
+    public void testIncludeMissingRowsMillionRowGap() throws Exception {
+        String rows = row("before") + emptyRows(1048565) + row("after");
+        for (boolean packaged : new boolean[]{false, true}) {
+            String xml = packaged
+                    ? autoDetect(ods(rows), "gap.ods", new Metadata(), 
odsWithMissingRows()).xml
+                    : autoDetect(fods(rows), "gap.fods", new Metadata(), 
fodsWithMissingRows()).xml;
+            assertEquals(100_000, xml.split("<tr>\\s*<td />\\s*</tr>", 
-1).length - 1);
+            assertTrue(xml.length() < 3_000_000, "output length " + 
xml.length());
+            assertContains("<p>after</p>", xml);
+        }
+    }
 }

Reply via email to