Xiao-zhen-Liu commented on code in PR #8769:
URL: https://github.com/apache/texera/pull/8769#discussion_r4213047473
##########
amber/src/main/python/core/runnables/data_processor.py:
##########
@@ -132,16 +141,40 @@ def _set_output_tuple(self, output_iterator:
Iterator[Optional[TupleLike]]) -> N
# output could be a None, a TupleLike, or a TableLike.
for output_tuple in all_output_to_tuple(output):
if output_tuple is not None:
- output_tuple.finalize(
- self._context.output_manager.get_port().get_schema()
- )
+ try:
+ output_tuple.finalize(
+
self._context.output_manager.get_port().get_schema()
+ )
+ except (KeyError, TypeError) as error:
+ # Only explain on failure, so valid rows are not
inspected.
+ raise self._explain_schema_mismatch(output, error)
from error
self._switch_context()
self._context.tuple_processing_manager.current_output_tuple = (
output_tuple
)
self._switch_context()
self._context.tuple_processing_manager.finished_current.set()
+ @staticmethod
+ def _explain_schema_mismatch(output, error: Exception) -> Exception:
+ """
+ Rebuild a schema check failure with a hint on how to fix the output.
+ A dict holding DataFrames is valid for BINARY columns (they are
+ pickled), so it is only called out once the schema check has failed.
+ """
+ if isinstance(output, dict) and any(
Review Comment:
This hint now fires whenever a row holding a DataFrame fails the schema
check, even when the DataFrame isn't the problem: with a BINARY column
`result`, `yield {"res": df}` (a misspelled column name) is told to "yield the
DataFrame itself" instead of getting the column hint. It should only fire when
a DataFrame sits in a column that isn't BINARY.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
