Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/22568#discussion_r220948939
--- Diff: python/pyspark/sql/tests.py ---
@@ -5714,24 +5755,31 @@ def test_wrong_args(self):
pandas_udf(lambda x, y: x, DoubleType(),
PandasUDFType.SCALAR))
def test_unsupported_types(self):
+ from distutils.version import LooseVersion
+ import pyarrow as pa
from pyspark.sql.functions import pandas_udf, PandasUDFType
- schema = StructType(
- [StructField("id", LongType(), True),
- StructField("map", MapType(StringType(), IntegerType()),
True)])
- with QuietTest(self.sc):
- with self.assertRaisesRegexp(
- NotImplementedError,
- 'Invalid returnType.*grouped map Pandas UDF.*MapType'):
- pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
- schema = StructType(
- [StructField("id", LongType(), True),
- StructField("arr_ts", ArrayType(TimestampType()), True)])
- with QuietTest(self.sc):
- with self.assertRaisesRegexp(
- NotImplementedError,
- 'Invalid returnType.*grouped map Pandas
UDF.*ArrayType.*TimestampType'):
- pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
+ common_err_msg = 'Invalid returnType.*grouped map Pandas UDF.*'
+ unsupported_types = [
+ StructField('map', MapType(StringType(), IntegerType())),
+ StructField('arr_ts', ArrayType(TimestampType())),
+ StructField('null', NullType()),
+ ]
+
+ # TODO: Remove this if-statement once minimum pyarrow version is
0.10.0
+ if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
+ unsupported_types.append(
+ StructField('bin', BinaryType())
+ )
+
+ for unsupported_type in unsupported_types:
+ schema = StructType([
+ StructField('id', LongType(), True),
+ unsupported_type
+ ])
--- End diff --
I think we can make this inlined as well.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
