This is an automated email from the ASF dual-hosted git repository. ruifengz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 3c8c7ad92cdc [SPARK-46255][PYTHON][CONNECT] Support complex type -> string conversion 3c8c7ad92cdc is described below commit 3c8c7ad92cdc3ae989456d87b0332cb917da7e4e Author: Ruifeng Zheng <ruife...@apache.org> AuthorDate: Tue Dec 5 13:05:36 2023 +0800 [SPARK-46255][PYTHON][CONNECT] Support complex type -> string conversion ### What changes were proposed in this pull request? Support complex type -> string conversion ### Why are the changes needed? to support `list -> str` conversion ### Does this PR introduce _any_ user-facing change? yes ### How was this patch tested? ci ### Was this patch authored or co-authored using generative AI tooling? no Closes #44171 from zhengruifeng/py_connect_str_conv. Authored-by: Ruifeng Zheng <ruife...@apache.org> Signed-off-by: Ruifeng Zheng <ruife...@apache.org> --- python/pyspark/sql/connect/conversion.py | 16 ---------------- python/pyspark/sql/tests/test_types.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/python/pyspark/sql/connect/conversion.py b/python/pyspark/sql/connect/conversion.py index 550978d02f85..fb5a2d4b17b1 100644 --- a/python/pyspark/sql/connect/conversion.py +++ b/python/pyspark/sql/connect/conversion.py @@ -222,22 +222,6 @@ class LocalDataToArrowConversion: if value is None: return None else: - # only atomic types are supported - assert isinstance( - value, - ( - bool, - int, - float, - str, - bytes, - bytearray, - decimal.Decimal, - datetime.date, - datetime.datetime, - datetime.timedelta, - ), - ) if isinstance(value, bool): # To match the PySpark which convert bool to string in # the JVM side (python.EvaluatePython.makeFromJava) diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py index b038cf6ce5ba..a07309d1dff9 100644 --- a/python/pyspark/sql/tests/test_types.py +++ b/python/pyspark/sql/tests/test_types.py @@ -507,6 +507,19 @@ class TypesTestsMixin: self.assertEqual(1, row.asDict()["l"][0].a) self.assertEqual(1.0, row.asDict()["d"]["key"].c) + def test_convert_list_to_str(self): + data = [[[123], 120]] + schema = StructType( + [ + StructField("name", StringType(), True), + StructField("income", LongType(), True), + ] + ) + df = self.spark.createDataFrame(data, schema) + self.assertEqual(df.schema, schema) + self.assertEqual(df.count(), 1) + self.assertEqual(df.head(), Row(name="[123]", income=120)) + def test_udt(self): from pyspark.sql.types import _parse_datatype_json_string, _infer_type, _make_type_verifier --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org