kaxil commented on code in PR #73926:
URL: https://github.com/apache/airflow/pull/73926#discussion_r4607050135
##########
providers/common/ai/src/airflow/providers/common/ai/operators/document_loader.py:
##########
@@ -450,21 +454,123 @@ def _parse_pdf_stream(self, stream: BinaryIO) ->
list[dict[str, Any]]:
documents.append({"text": text, "metadata": {"page_number":
page_num + 1}})
return documents
- def _parse_docx_stream(self, stream: BinaryIO) -> list[dict[str, Any]]:
+ def _parse_docx_stream(self, stream: BinaryIO, *, source_hint: str) ->
list[dict[str, Any]]:
"""
- Parse a DOCX stream into documents.
+ Parse a DOCX stream into a single document.
- Extracts paragraph text only. Tables, headers, footers, and footnotes
- are not included. For richer DOCX parsing, plug in a dedicated
- extraction tool (``Unstructured``, ``docling``) as a custom parser
- backend.
+ Paragraphs and tables in the document body are extracted in document
+ order. Each table row becomes one "| cell | cell |" line, and a nested
+ table is flattened into its cell. Headers, footers, footnotes, content
+ controls, text boxes, and pending tracked insertions are not included.
"""
try:
from docx import Document
+ from docx.oxml.exceptions import InvalidXmlError
+ from docx.table import Table
except ImportError as e:
raise AirflowOptionalProviderFeatureException(e)
doc = Document(stream)
- paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
- text = "\n\n".join(paragraphs)
+ blocks = []
+ for block in doc.iter_inner_content():
+ if isinstance(block, Table):
+ rows = self._get_docx_table_rows_or_none(
+ block, Table, InvalidXmlError, source_hint=source_hint
+ )
+ if rows is None:
+ continue
+ text = "\n".join(f"| {' | '.join(cells)} |" for cells in rows)
+ else:
+ text = block.text
+ if text.strip():
+ blocks.append(text)
+ text = "\n\n".join(blocks)
return [{"text": text, "metadata": {}}]
+
+ def _get_docx_table_rows_or_none(
+ self,
+ table: Table,
+ table_cls: type[Table],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> list[list[str]] | None:
+ try:
+ return self._get_docx_table_rows(table, table_cls,
invalid_xml_error_cls, source_hint=source_hint)
+ except (ValueError, invalid_xml_error_cls, RecursionError) as e:
+ # Malformed table XML raises ValueError (a vMerge with no cell
above, or a gridSpan
+ # that does not fit the table) or InvalidXmlError (a missing
required attribute).
+ # python-docx walks vertical merges up recursively, so one
spanning about 1000 rows
+ # raises RecursionError.
+ self.log.warning("Skipping a table in %s that python-docx could
not read: %r", source_hint, e)
+ return None
+
+ def _get_docx_table_rows(
+ self,
+ table: Table,
+ table_cls: type[Table],
+ invalid_xml_error_cls: type[InvalidXmlError],
+ *,
+ source_hint: str,
+ ) -> list[list[str]]:
+ # w:gridBefore and w:gridAfter come straight from the XML and can be
any integer, so the
+ # empty cells added for them only fill the row up to the column count.
+ column_count = len(table.columns)
+
+ rows = []
+ for row in table.rows:
+ # row.cells yields a cell once for every grid column its
w:gridSpan covers, all at
+ # once: spans that add up to more than the table width are built
in full, and a span
+ # below 1 drops the cell and its text. Both are rejected before
row.cells is read.
+ spanned_columns = 0
+ for index, tc in enumerate(row._tr.tc_lst, start=1):
+ spanned_columns += tc.grid_span
+ if tc.grid_span < 1 or spanned_columns > column_count:
+ raise ValueError(
+ f"w:gridSpan {tc.grid_span} of cell {index} does not
fit the {column_count} table columns"
+ )
+ cells: list[str] = []
+ previous_cell = None
+ for cell in row.cells:
Review Comment:
Per-table is fine for this PR, and a follow-up issue for the document-wide
budget together with the decompressed-size check sounds good. One thing for
that issue: the budget has to be counted per cell rather than per row, since a
cell with many nested tables currently builds all of them before the parent's
row is counted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]