This is an automated email from the ASF dual-hosted git repository. xinrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e578d466d4e [SPARK-44446][PYTHON] Add checks for expected list type special cases e578d466d4e is described below commit e578d466d4eae808a8ad5e42681b9e3e87fe6ca7 Author: Amanda Liu <amanda....@databricks.com> AuthorDate: Mon Jul 17 11:43:05 2023 -0700 [SPARK-44446][PYTHON] Add checks for expected list type special cases ### What changes were proposed in this pull request? This PR adds handling for special cases when `expected` is type list. ### Why are the changes needed? The change is needed to handle all cases for when `expected` is type list. ### Does this PR introduce _any_ user-facing change? Yes, the PR makes modifications to the user-facing function `assertDataFrameEqual` ### How was this patch tested? Added tests to `runtime/python/pyspark/sql/tests/test_utils.py` and `runtime/python/pyspark/sql/tests/connect/test_utils.py` Closes #42023 from asl3/fix-list-support. Authored-by: Amanda Liu <amanda....@databricks.com> Signed-off-by: Xinrong Meng <xinr...@apache.org> --- python/pyspark/sql/tests/test_utils.py | 24 ++++++++++++++++++++++++ python/pyspark/testing/utils.py | 15 +++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py index 5b859ad15a5..eae3f528504 100644 --- a/python/pyspark/sql/tests/test_utils.py +++ b/python/pyspark/sql/tests/test_utils.py @@ -1119,6 +1119,30 @@ class UtilsTestsMixin: assertDataFrameEqual(df1, df2, checkRowOrder=False) assertDataFrameEqual(df1, df2, checkRowOrder=True) + def test_empty_expected_list(self): + df1 = self.spark.range(0, 10).drop("id") + + df2 = [] + + assertDataFrameEqual(df1, df2, checkRowOrder=False) + assertDataFrameEqual(df1, df2, checkRowOrder=True) + + def test_no_column_expected_list(self): + df1 = self.spark.range(0, 10).limit(0) + + df2 = [] + + assertDataFrameEqual(df1, df2, checkRowOrder=False) + assertDataFrameEqual(df1, df2, checkRowOrder=True) + + def test_empty_no_column_expected_list(self): + df1 = self.spark.range(0, 10).drop("id").limit(0) + + df2 = [] + + assertDataFrameEqual(df1, df2, checkRowOrder=False) + assertDataFrameEqual(df1, df2, checkRowOrder=True) + def test_special_vals(self): df1 = self.spark.createDataFrame( data=[ diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py index 21c7b7e4dcd..14db9264209 100644 --- a/python/pyspark/testing/utils.py +++ b/python/pyspark/testing/utils.py @@ -349,6 +349,8 @@ def assertDataFrameEqual( For checkRowOrder, note that PySpark DataFrame ordering is non-deterministic, unless explicitly sorted. + Note that schema equality is checked only when `expected` is a DataFrame (not a list of Rows). + For DataFrames with float values, assertDataFrame asserts approximate equality. Two float values a and b are approximately equal if the following equation is True: @@ -362,6 +364,9 @@ def assertDataFrameEqual( >>> df1 = spark.createDataFrame(data=[("1", 0.1), ("2", 3.23)], schema=["id", "amount"]) >>> df2 = spark.createDataFrame(data=[("1", 0.109), ("2", 3.23)], schema=["id", "amount"]) >>> assertDataFrameEqual(df1, df2, rtol=1e-1) # pass, DataFrames are approx equal by rtol + >>> df1 = spark.createDataFrame(data=[(1, 1000), (2, 3000)], schema=["id", "amount"]) + >>> list_of_rows = [Row(1, 1000), Row(2, 3000)] + >>> assertDataFrameEqual(df1, list_of_rows) # pass, actual and expected are equal >>> df1 = spark.createDataFrame( ... data=[("1", 1000.00), ("2", 3000.00), ("3", 2000.00)], schema=["id", "amount"]) >>> df2 = spark.createDataFrame( @@ -415,8 +420,14 @@ def assertDataFrameEqual( ) # special cases: empty datasets, datasets with 0 columns - if (actual.first() is None and expected.first() is None) or ( - len(actual.columns) == 0 and len(expected.columns) == 0 + if ( + isinstance(expected, DataFrame) + and ( + (actual.first() is None and expected.first() is None) + or (len(actual.columns) == 0 and len(expected.columns) == 0) + ) + or isinstance(expected, list) + and ((actual.first() is None or len(actual.columns) == 0) and len(expected) == 0) ): return True --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org