Github user AlexanderKoryagin commented on a diff in the pull request:

    https://github.com/apache/spark/pull/22568#discussion_r221186913
  
    --- Diff: python/pyspark/sql/tests.py ---
    @@ -5714,24 +5755,31 @@ def test_wrong_args(self):
                         pandas_udf(lambda x, y: x, DoubleType(), 
PandasUDFType.SCALAR))
     
         def test_unsupported_types(self):
    +        from distutils.version import LooseVersion
    +        import pyarrow as pa
             from pyspark.sql.functions import pandas_udf, PandasUDFType
    -        schema = StructType(
    -            [StructField("id", LongType(), True),
    -             StructField("map", MapType(StringType(), IntegerType()), 
True)])
    -        with QuietTest(self.sc):
    -            with self.assertRaisesRegexp(
    -                    NotImplementedError,
    -                    'Invalid returnType.*grouped map Pandas UDF.*MapType'):
    -                pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
     
    -        schema = StructType(
    -            [StructField("id", LongType(), True),
    -             StructField("arr_ts", ArrayType(TimestampType()), True)])
    -        with QuietTest(self.sc):
    -            with self.assertRaisesRegexp(
    -                    NotImplementedError,
    -                    'Invalid returnType.*grouped map Pandas 
UDF.*ArrayType.*TimestampType'):
    -                pandas_udf(lambda x: x, schema, PandasUDFType.GROUPED_MAP)
    +        common_err_msg = 'Invalid returnType.*grouped map Pandas UDF.*'
    +        unsupported_types = [
    +            StructField('map', MapType(StringType(), IntegerType())),
    +            StructField('arr_ts', ArrayType(TimestampType())),
    +            StructField('null', NullType()),
    +        ]
    +
    +        # TODO: Remove this if-statement once minimum pyarrow version is 
0.10.0
    +        if LooseVersion(pa.__version__) < LooseVersion("0.10.0"):
    +            unsupported_types.append(
    +                StructField('bin', BinaryType())
    --- End diff --
    
    Done


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to