Spenserrrr commented on code in PR #58751:
URL: https://github.com/apache/spark/pull/58751#discussion_r4010236982
##########
python/pyspark/sql/conversion.py:
##########
@@ -410,105 +401,159 @@ def convert_column(
assign_cols_by_name=assign_cols_by_name,
int_to_decimal_coercion_enabled=int_to_decimal_coercion_enabled,
ignore_unexpected_complex_type_values=ignore_unexpected_complex_type_values,
- is_legacy=is_legacy,
+ use_legacy_error_handling=use_legacy_error_handling,
)
# Wrap the nested RecordBatch as a single StructArray column
- return
ArrowBatchTransformer.wrap_struct(nested_batch).column(0)
+
converted.append(ArrowBatchTransformer.wrap_struct(nested_batch).column(0))
+ else:
+ converted.append(
+ cls._convert_column(
+ col,
+ field,
+ timezone=timezone,
+ safecheck=safecheck,
+ arrow_cast=arrow_cast,
+ prefers_large_types=prefers_large_types,
+
int_to_decimal_coercion_enabled=int_to_decimal_coercion_enabled,
+
ignore_unexpected_complex_type_values=ignore_unexpected_complex_type_values,
+ use_legacy_error_handling=use_legacy_error_handling,
+ )
+ )
- series = col
- field_name = field.name
- ret_type = field.dataType
+ # pa.Array.from_pandas returns a pa.ChunkedArray for a chunked
arrow-backed Series
+ # (e.g. a pyarrow-backed extension dtype), which
pa.RecordBatch.from_arrays rejects.
+ arrays = [a.combine_chunks() if isinstance(a, pa.ChunkedArray) else a
for a in converted]
+ return pa.RecordBatch.from_arrays(arrays, schema.names)
- if isinstance(series.dtype, pd.CategoricalDtype):
- series = series.astype(series.dtype.categories.dtype)
+ @classmethod
+ def _convert_column(
+ cls,
+ series: "pd.Series",
+ field: StructField,
+ *,
+ timezone: Optional[str] = None,
+ safecheck: bool = True,
+ arrow_cast: bool = False,
+ prefers_large_types: bool = False,
+ int_to_decimal_coercion_enabled: bool = False,
+ ignore_unexpected_complex_type_values: bool = False,
+ use_legacy_error_handling: bool = False,
+ ) -> Union["pa.Array", "pa.ChunkedArray"]:
+ """Dispatch a pandas column to its conversion strategy."""
Review Comment:
Added a comment clarifying why this currently delegates only to the legacy
implementation. Planned follow-ups will migrate applicable types to
_convert_column_numpy, following the read leg, and add _convert_column_pyarrow
for Arrow-backed pandas Series.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
