xlsongc commented on code in PR #73926:
URL: https://github.com/apache/airflow/pull/73926#discussion_r4424560223
##########
providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py:
##########
@@ -450,21 +454,123 @@ def _parse_pdf_stream(self, stream: BinaryIO) ->
list[dict[str, Any]]:
documents.append({"text": text, "metadata": {"page_number":
page_num + 1}})
return documents
- def _parse_docx_stream(self, stream: BinaryIO) -> list[dict[str, Any]]:
+ def _parse_docx_stream(self, stream: BinaryIO, *, source_hint: str) ->
list[dict[str, Any]]:
"""
- Parse a DOCX stream into documents.
+ Parse a DOCX stream into a single document.
- Extracts paragraph text only. Tables, headers, footers, and footnotes
- are not included. For richer DOCX parsing, plug in a dedicated
- extraction tool (``Unstructured``, ``docling``) as a custom parser
- backend.
+ Paragraphs and tables in the document body are extracted in document
+ order. Each table row becomes one "| cell | cell |" line, and a nested
+ table is flattened into its cell. Headers, footers, footnotes, content
+ controls, text boxes, and pending tracked insertions are not included.
"""
try:
from docx import Document
+ from docx.oxml.exceptions import InvalidXmlError
+ from docx.table import Table
except ImportError as e:
raise AirflowOptionalProviderFeatureException(e)
doc = Document(stream)
- paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
- text = "\n\n".join(paragraphs)
+ blocks = []
+ for block in doc.iter_inner_content():
+ if isinstance(block, Table):
+ rows = self._get_docx_table_rows_or_none(
+ block, Table, InvalidXmlError, source_hint=source_hint
+ )
+ if rows is None:
+ continue
+ text = "\n".join(f"| {' | '.join(cells)} |" for cells in rows)
+ else:
+ text = block.text
+ if text.strip():
+ blocks.append(text)
+ text = "\n\n".join(blocks)
return [{"text": text, "metadata": {}}]
+
+ def _get_docx_table_rows_or_none(
+ self,
+ table: Table,
+ table_cls: type[Table],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> list[list[str]] | None:
+ try:
+ return self._get_docx_table_rows(table, table_cls,
invalid_xml_error_cls, source_hint=source_hint)
+ except (ValueError, invalid_xml_error_cls, RecursionError) as e:
+ # Malformed table XML raises ValueError (a vMerge with no cell
above, or a gridSpan
+ # that does not fit the table) or InvalidXmlError (a missing
required attribute).
+ # python-docx walks vertical merges up recursively, so one
spanning about 1000 rows
+ # raises RecursionError.
+ self.log.warning("Skipping a table in %s that python-docx could
not read: %r", source_hint, e)
+ return None
+
+ def _get_docx_table_rows(
+ self,
+ table: Table,
+ table_cls: type[Table],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> list[list[str]]:
+ # w:gridBefore and w:gridAfter come straight from the XML and can be
any integer, so the
+ # empty cells added for them only fill the row up to the column count.
+ column_count = len(table.columns)
Review Comment:
Done. When `w:tblGrid` is missing, python-docx's `InvalidXmlError` is caught
and the widest row sets the column count: `w:gridBefore` + spans +
`w:gridAfter`, with negative values floored at 0 as in the row loop, and 0 for
a table without rows. The width is then capped at `_DOCX_TABLE_MAX_COLUMNS =
1000` with or without a grid, and a wider table is skipped with a warning, so
your 5000-column file is skipped. At 1000 columns the padding still counts
toward the text cap (3 characters per empty cell), so 2000 such rows are
skipped too.
On the number: Word's Insert Table allows at most 63 columns, and a
hand-built row with more than 63 `w:tc` doesn't open in Word. Word can still
save more grid columns than that, though: a 40-cell row above a 41-cell row
saved 80 `w:gridCol`. Grid columns can grow with the row count, so no fixed cap
covers every file Word opens; 1000 is well above real tables.
`test_docx_table_without_grid_sized_by_widest_row`,
`test_docx_table_without_grid_ignores_negative_grid_values` and
`test_docx_table_without_grid_or_rows_left_out_without_warning` cover the
fallback, and `test_docx_table_wider_than_limit_skipped_with_warning` keeps a
1000-column table and skips a 1001-column one, with and without a grid. Each
fails without the change, and the tests pass on python-docx 1.1.2 and 1.2.0.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
