srielau commented on code in PR #58549:
URL: https://github.com/apache/spark/pull/58549#discussion_r4049079996
##########
python/pyspark/sql/session.py:
##########
@@ -1611,6 +1617,13 @@ def createDataFrame( # type: ignore[misc]
elif isinstance(schema, (list, tuple)):
# Must re-encode any unicode strings to be consistent with
StructField names
schema = [x.encode("utf-8") if not isinstance(x, str) else x for x
in schema]
+ if isinstance(schema, DataType) and _has_char_varchar_in_udt(schema):
Review Comment:
Added validation of the finalized schema in both `_createFromRDD` and
`_createFromLocal`, before conversion, while retaining early explicit-schema
validation. Added explicit and inferred UDT rejection coverage in Classic and
Connect.
##########
python/pyspark/sql/session.py:
##########
@@ -1611,6 +1617,13 @@ def createDataFrame( # type: ignore[misc]
elif isinstance(schema, (list, tuple)):
# Must re-encode any unicode strings to be consistent with
StructField names
schema = [x.encode("utf-8") if not isinstance(x, str) else x for x
in schema]
+ if isinstance(schema, DataType) and _has_char_varchar_in_udt(schema):
Review Comment:
Added validation of the finalized schema in both `_createFromRDD` and
`_createFromLocal`, before conversion, while retaining early explicit-schema
validation. Added explicit and inferred UDT rejection coverage in Classic and
Connect.
##########
python/pyspark/sql/types.py:
##########
@@ -2877,8 +2877,38 @@ def _has_type(dt: DataType, dts: Union[type, Tuple[type,
...]]) -> bool:
return _has_type(dt.elementType, dts)
elif isinstance(dt, MapType):
return _has_type(dt.keyType, dts) or _has_type(dt.valueType, dts)
+ else:
Review Comment:
Restored `_has_type`'s prior physical recursion and made
`_has_physical_type` delegate to it. Logical UDF capability checks now use a
separate `_has_logical_type` helper.
##########
python/pyspark/sql/types.py:
##########
@@ -2877,8 +2877,38 @@ def _has_type(dt: DataType, dts: Union[type, Tuple[type,
...]]) -> bool:
return _has_type(dt.elementType, dts)
elif isinstance(dt, MapType):
return _has_type(dt.keyType, dts) or _has_type(dt.valueType, dts)
+ else:
Review Comment:
Restored `_has_type`'s prior physical recursion and made
`_has_physical_type` delegate to it. Logical UDF capability checks now use a
separate `_has_logical_type` helper.
##########
python/pyspark/sql/tests/arrow/test_arrow_udf_scalar.py:
##########
@@ -58,6 +60,50 @@
@unittest.skipIf(not have_pyarrow, pyarrow_requirement_message)
class ScalarArrowUDFTestsMixin:
+ def test_char_varchar_scalar_results(self):
+ import pyarrow as pa
+
+ @arrow_udf(CharType(3), ArrowUDFType.SCALAR)
+ def scalar_char(values):
+ return pa.array(["a"] * len(values))
+
+ @arrow_udf(CharType(3), ArrowUDFType.SCALAR_ITER)
+ def iterator_char(batches):
+ for values in batches:
+ yield pa.array(["a"] * len(values))
+
+ @arrow_udf(VarcharType(3), ArrowUDFType.SCALAR)
+ def scalar_varchar(values):
+ return pa.array(["abcd"] * len(values))
+
+ @arrow_udf(VarcharType(3), ArrowUDFType.SCALAR_ITER)
+ def iterator_varchar(batches):
+ for values in batches:
+ yield pa.array(["abcd"] * len(values))
+
+ with self.sql_conf(
Review Comment:
Completed the standard/default/legacy coverage across native Arrow
scalar/iterator, pandas scalar/iterator, HOF, `toArrow`, and pandas/PyArrow
creation paths, including non-iterator pandas scalar coverage.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/python/ArrowColumnarPythonUDFSuite.scala:
##########
@@ -103,6 +111,181 @@ class ArrowColumnarPythonUDFSuite extends
SharedSparkSession {
}
}
+ test("Arrow-backed source: CHAR/VARCHAR output checks") {
Review Comment:
Added an Arrow-backed combined regression with a nested checked CHAR result,
an unchecked VARCHAR sibling, and a checked VARCHAR sibling. It asserts output
schemas and ordinals, padding, unchecked overflow acceptance, and checked
overflow rejection.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
