asfgit closed pull request #23305: [SPARK-26355][PYSPARK] Add a workaround for
PyArrow 0.11.
URL: https://github.com/apache/spark/pull/23305
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index f3ebd3767a0a1..fd4695210fb7c 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -281,7 +281,10 @@ def create_array(s, t):
# TODO: see ARROW-2432. Remove when the minimum PyArrow version
becomes 0.10.0.
return pa.Array.from_pandas(s.apply(
lambda v: decimal.Decimal('NaN') if v is None else v),
mask=mask, type=t)
- return pa.Array.from_pandas(s, mask=mask, type=t)
+ elif LooseVersion(pa.__version__) < LooseVersion("0.11.0"):
+ # TODO: see ARROW-1949. Remove when the minimum PyArrow version
becomes 0.11.0.
+ return pa.Array.from_pandas(s, mask=mask, type=t)
+ return pa.Array.from_pandas(s, mask=mask, type=t, safe=False)
arrs = [create_array(s, t) for s, t in series]
return pa.RecordBatch.from_arrays(arrs, ["_%d" % i for i in
xrange(len(arrs))])
diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
b/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
index bfecc071386e9..a12c608dff9dd 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
@@ -468,8 +468,15 @@ def invalid_positional_types(pdf):
with QuietTest(self.sc):
with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
grouped_df.apply(column_name_typo).collect()
- with self.assertRaisesRegexp(Exception, "No cast implemented"):
- grouped_df.apply(invalid_positional_types).collect()
+ from distutils.version import LooseVersion
+ import pyarrow as pa
+ if LooseVersion(pa.__version__) < LooseVersion("0.11.0"):
+ # TODO: see ARROW-1949. Remove when the minimum PyArrow
version becomes 0.11.0.
+ with self.assertRaisesRegexp(Exception, "No cast implemented"):
+ grouped_df.apply(invalid_positional_types).collect()
+ else:
+ with self.assertRaisesRegexp(Exception, "an integer is
required"):
+ grouped_df.apply(invalid_positional_types).collect()
def test_positional_assignment_conf(self):
import pandas as pd
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
