Github user HyukjinKwon commented on a diff in the pull request: https://github.com/apache/spark/pull/22568#discussion_r221194037 --- Diff: python/pyspark/sql/tests.py --- @@ -5525,32 +5525,73 @@ def data(self): .withColumn("v", explode(col('vs'))).drop('vs') def test_supported_types(self): - from pyspark.sql.functions import pandas_udf, PandasUDFType, array, col - df = self.data.withColumn("arr", array(col("id"))) + from decimal import Decimal + from distutils.version import LooseVersion + import pyarrow as pa + from pyspark.sql.functions import pandas_udf, PandasUDFType - # Different forms of group map pandas UDF, results of these are the same + input_values_with_schema = [ + (1, StructField('id', IntegerType())), + (2, StructField('byte', ByteType())), + (3, StructField('short', ShortType())), + (4, StructField('int', IntegerType())), + (5, StructField('long', LongType())), + (1.1, StructField('float', FloatType())), + (2.2, StructField('double', DoubleType())), + (Decimal(1.123), StructField('decim', DecimalType(10, 3))), + ([1, 2, 3], StructField('array', ArrayType(IntegerType()))), + (True, StructField('bool', BooleanType())), + ('hello', StructField('str', StringType())), + ] --- End diff -- Eh, how about: ``` values = [ 1, 2, 3, 4, 5, 1.1, ... ] output_fields = [ ('id', IntegerType()), ('byte', ByteType()), ('short', ShortType()), ('int', IntegerType()), ('long', LongType()), ('float', FloatType()), ... ]) output_schema = StructType(map(lambda x: StructField(*x), output_fields)) ```
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org