This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 224fd35d6fca Revert "[SPARK-48250][PYTHON][CONNECT][TESTS] Enable array inference tests at test_parity_types.py" 224fd35d6fca is described below commit 224fd35d6fca63ed0f321817ea8c35620abf8caf Author: Hyukjin Kwon <gurwls...@apache.org> AuthorDate: Mon May 13 20:13:52 2024 +0900 Revert "[SPARK-48250][PYTHON][CONNECT][TESTS] Enable array inference tests at test_parity_types.py" This reverts commit 13b0d1aab36740293814ce54e38cb4d86f8b762d. --- python/pyspark/sql/tests/connect/test_parity_types.py | 8 ++++++-- python/pyspark/sql/tests/test_types.py | 14 +++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/python/pyspark/sql/tests/connect/test_parity_types.py b/python/pyspark/sql/tests/connect/test_parity_types.py index 55acb4b1a381..82a677574b45 100644 --- a/python/pyspark/sql/tests/connect/test_parity_types.py +++ b/python/pyspark/sql/tests/connect/test_parity_types.py @@ -39,8 +39,12 @@ class TypesParityTests(TypesTestsMixin, ReusedConnectTestCase): super().test_create_dataframe_schema_mismatch() @unittest.skip("Spark Connect does not support RDD but the tests depend on them.") - def test_infer_array_element_type_empty_rdd(self): - super().test_infer_array_element_type_empty_rdd() + def test_infer_array_element_type_empty(self): + super().test_infer_array_element_type_empty() + + @unittest.skip("Spark Connect does not support RDD but the tests depend on them.") + def test_infer_array_element_type_with_struct(self): + super().test_infer_array_element_type_with_struct() @unittest.skip("Spark Connect does not support RDD but the tests depend on them.") def test_infer_array_merge_element_types_with_rdd(self): diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py index bd99804ec565..159678937683 100644 --- a/python/pyspark/sql/tests/test_types.py +++ b/python/pyspark/sql/tests/test_types.py @@ -366,7 +366,7 @@ class TypesTestsMixin: df = self.spark.createDataFrame(rdd) self.assertEqual(Row(f1=[1, None], f2=[None, 2]), df.first()) - def test_infer_array_element_type_empty_rdd(self): + def test_infer_array_element_type_empty(self): # SPARK-39168: Test inferring array element type from all rows ArrayRow = Row("f1") @@ -379,12 +379,6 @@ class TypesTestsMixin: self.assertEqual(Row(f1=[None]), rows[1]) self.assertEqual(Row(f1=[1]), rows[2]) - def test_infer_array_element_type_empty(self): - # SPARK-39168: Test inferring array element type from all rows - ArrayRow = Row("f1") - - data = [ArrayRow([]), ArrayRow([None]), ArrayRow([1])] - df = self.spark.createDataFrame(data) rows = df.collect() self.assertEqual(Row(f1=[]), rows[0]) @@ -398,6 +392,12 @@ class TypesTestsMixin: with self.sql_conf({"spark.sql.pyspark.inferNestedDictAsStruct.enabled": True}): data = [NestedRow([{"payment": 200.5}, {"name": "A"}])] + nestedRdd = self.sc.parallelize(data) + df = self.spark.createDataFrame(nestedRdd) + self.assertEqual( + Row(f1=[Row(payment=200.5, name=None), Row(payment=None, name="A")]), df.first() + ) + df = self.spark.createDataFrame(data) self.assertEqual( Row(f1=[Row(payment=200.5, name=None), Row(payment=None, name="A")]), df.first() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org