This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 02bbd2867be [SPARK-43628][SPARK-43629][CONNECT][PS][TESTS] Clear
message for JVM dependent tests
02bbd2867be is described below
commit 02bbd2867be52187239074f13617ef0ba2acf09e
Author: Haejoon Lee <haejoon....@databricks.com>
AuthorDate: Mon Sep 18 09:02:52 2023 -0700
[SPARK-43628][SPARK-43629][CONNECT][PS][TESTS] Clear message for JVM
dependent tests
### What changes were proposed in this pull request?
This PR proposes to correct the message for JVM only tests from Spark
Connect, and enable the tests when possible to workaround without JVM features.
### Why are the changes needed?
Among JVM-dependent tests, there are tests that can be replaced without
using JVM features, while there are some edge tests that can only be tested
using JVM features. We need to be clearer about why these cannot be tested.
### Does this PR introduce _any_ user-facing change?
No. it's test-only
### How was this patch tested?
Updated the existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #42955 from itholic/connect_jvm_tests.
Authored-by: Haejoon Lee <haejoon....@databricks.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../pandas/tests/connect/groupby/test_parity_apply_func.py | 2 +-
.../tests/connect/plot/test_parity_frame_plot_matplotlib.py | 2 +-
.../pandas/tests/connect/plot/test_parity_frame_plot_plotly.py | 2 +-
.../tests/connect/plot/test_parity_series_plot_matplotlib.py | 2 +-
.../pandas/tests/connect/plot/test_parity_series_plot_plotly.py | 2 +-
python/pyspark/pandas/tests/connect/test_parity_frame_spark.py | 8 ++++----
6 files changed, 9 insertions(+), 9 deletions(-)
diff --git
a/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
b/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
index 2c0d49ebb5c..4daf84bd1b5 100644
--- a/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
+++ b/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
@@ -24,7 +24,7 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class GroupbyParityApplyFuncTests(
GroupbyApplyFuncMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43629): Enable RDD dependent tests with Spark
Connect.")
+ @unittest.skip("Test depends on SparkContext which is not supported from
Spark Connect.")
def test_apply_with_side_effect(self):
super().test_apply_with_side_effect()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
index 98da8858a03..3f615326f2b 100644
---
a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
+++
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
@@ -28,7 +28,7 @@ class DataFramePlotMatplotlibParityTests(
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-43629): Enable RDD with Spark Connect.")
+ @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
index 7a3efee06df..16b97d6814e 100644
--- a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
+++ b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
@@ -32,7 +32,7 @@ class DataFramePlotPlotlyParityTests(
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-43629): Enable RDD with Spark Connect.")
+ @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
index 975d78f17c6..9e264e76229 100644
---
a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
+++
b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
@@ -32,7 +32,7 @@ class SeriesPlotMatplotlibParityTests(
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-43629): Enable RDD with Spark Connect.")
+ @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
index 5ced8cb226e..7d6c92e56ac 100644
--- a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
+++ b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
@@ -28,7 +28,7 @@ class SeriesPlotPlotlyParityTests(
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-43629): Enable RDD with Spark Connect.")
+ @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git a/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py
b/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py
index 986ddde4db0..24626a9164e 100644
--- a/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py
+++ b/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py
@@ -24,19 +24,19 @@ from pyspark.testing.pandasutils import
PandasOnSparkTestUtils, TestUtils
class SparkFrameMethodsParityTests(
SparkFrameMethodsTestsMixin, TestUtils, PandasOnSparkTestUtils,
ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43628): Enable SparkContext with Spark
Connect.")
+ @unittest.skip("Test depends on checkpoint which is not supported from
Spark Connect.")
def test_checkpoint(self):
super().test_checkpoint()
- @unittest.skip("TODO(SPARK-43629): Enable RDD with Spark Connect.")
+ @unittest.skip("Test depends on RDD which is not supported from Spark
Connect.")
def test_coalesce(self):
super().test_coalesce()
- @unittest.skip("TODO(SPARK-43630): Implement `localCheckpoint` for Spark
Connect DataFrame.")
+ @unittest.skip("Test depends on localCheckpoint which is not supported
from Spark Connect.")
def test_local_checkpoint(self):
super().test_local_checkpoint()
- @unittest.skip("TODO(SPARK-43629): Enable RDD with Spark Connect.")
+ @unittest.skip("Test depends on RDD which is not supported from Spark
Connect.")
def test_repartition(self):
super().test_repartition()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
