This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 4bd975fa79a [SPARK-44597][PYTHON][TESTS] Migrate test_sql assert_eq to
use assertDataFrameEqual
4bd975fa79a is described below
commit 4bd975fa79a82a417bc9639dcc3305f5624ac861
Author: Amanda Liu <amanda....@databricks.com>
AuthorDate: Mon Jul 31 09:03:20 2023 +0900
[SPARK-44597][PYTHON][TESTS] Migrate test_sql assert_eq to use
assertDataFrameEqual
### What changes were proposed in this pull request?
This PR updates the `python/pyspark/pandas/tests/test_sql.py` to use the
new PySpark test util function, `assertDataFrameEqual`, introduced in
[SPARK-44042](https://issues.apache.org/jira/browse/SPARK-44042).
### Why are the changes needed?
This change is needed as part of the effort in
[SPARK-44589](https://issues.apache.org/jira/browse/SPARK-44589) to unify the
tests in the codebase to use new PySpark test utils.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests in `python/pyspark/pandas/tests/test_sql.py`.
Closes #42217 from asl3/migrate-test-sql.
Authored-by: Amanda Liu <amanda....@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
(cherry picked from commit 9f03f434358d98e87cd80228ea59f05935a37b9f)
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
python/pyspark/pandas/tests/test_sql.py | 25 ++++++++++++++-----------
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/python/pyspark/pandas/tests/test_sql.py
b/python/pyspark/pandas/tests/test_sql.py
index ec56fe210f9..7800d5756b0 100644
--- a/python/pyspark/pandas/tests/test_sql.py
+++ b/python/pyspark/pandas/tests/test_sql.py
@@ -19,6 +19,7 @@ from pyspark import pandas as ps
from pyspark.errors import ParseException
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.testing.sqlutils import SQLTestUtils
+from pyspark.testing.utils import assertDataFrameEqual
class SQLTestsMixin:
@@ -49,7 +50,7 @@ class SQLTestsMixin:
psdf_reset_index=psdf_reset_index,
)
expected = psdf.iloc[[1, 2]]
- self.assert_eq(actual, expected)
+ assertDataFrameEqual(actual, expected)
# MultiIndex
psdf = ps.DataFrame(
@@ -65,27 +66,29 @@ class SQLTestsMixin:
psdf_reset_index=psdf_reset_index,
)
expected = psdf.iloc[[1, 2]]
- self.assert_eq(actual, expected)
+ assertDataFrameEqual(actual, expected)
def test_sql_with_pandas_objects(self):
import pandas as pd
pdf = pd.DataFrame({"a": [1, 2, 3, 4]})
- self.assert_eq(ps.sql("SELECT {col} + 1 as a FROM {tbl}", col=pdf.a,
tbl=pdf), pdf + 1)
+ assertDataFrameEqual(
+ ps.sql("SELECT {col} + 1 as a FROM {tbl}", col=pdf.a, tbl=pdf),
pdf + 1
+ )
def test_sql_with_python_objects(self):
- self.assert_eq(
+ assertDataFrameEqual(
ps.sql("SELECT {col} as a FROM range(1)", col="lit"),
ps.DataFrame({"a": ["lit"]})
)
- self.assert_eq(
+ assertDataFrameEqual(
ps.sql("SELECT id FROM range(10) WHERE id IN {pred}", col="lit",
pred=(1, 2, 3)),
ps.DataFrame({"id": [1, 2, 3]}),
)
- self.assert_eq(
+ assertDataFrameEqual(
ps.sql("SELECT {col} as a FROM range(1)", col="a'''c''d"),
ps.DataFrame({"a": ["a'''c''d"]}),
)
- self.assert_eq(
+ assertDataFrameEqual(
ps.sql("SELECT id FROM range(10) WHERE id IN {pred}",
col="a'''c''d", pred=(1, 2, 3)),
ps.DataFrame({"id": [1, 2, 3]}),
)
@@ -93,14 +96,14 @@ class SQLTestsMixin:
def test_sql_with_pandas_on_spark_objects(self):
psdf = ps.DataFrame({"a": [1, 2, 3, 4]})
- self.assert_eq(ps.sql("SELECT {col} FROM {tbl}", col=psdf.a,
tbl=psdf), psdf)
- self.assert_eq(ps.sql("SELECT {tbl.a} FROM {tbl}", tbl=psdf), psdf)
+ assertDataFrameEqual(ps.sql("SELECT {col} FROM {tbl}", col=psdf.a,
tbl=psdf), psdf)
+ assertDataFrameEqual(ps.sql("SELECT {tbl.a} FROM {tbl}", tbl=psdf),
psdf)
psdf = ps.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
- self.assert_eq(
+ assertDataFrameEqual(
ps.sql("SELECT {col}, {col2} FROM {tbl}", col=psdf.A, col2=psdf.B,
tbl=psdf), psdf
)
- self.assert_eq(ps.sql("SELECT {tbl.A}, {tbl.B} FROM {tbl}", tbl=psdf),
psdf)
+ assertDataFrameEqual(ps.sql("SELECT {tbl.A}, {tbl.B} FROM {tbl}",
tbl=psdf), psdf)
class SQLTests(SQLTestsMixin, PandasOnSparkTestCase, SQLTestUtils):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
