This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 36e626bc60a [SPARK-43711][SPARK-44372][CONNECT][PS][TESTS] Clear
message for Spark ML dependent tests
36e626bc60a is described below
commit 36e626bc60af4ce94a1ca304e05390418b965135
Author: Haejoon Lee <haejoon....@databricks.com>
AuthorDate: Mon Sep 25 11:23:51 2023 -0700
[SPARK-43711][SPARK-44372][CONNECT][PS][TESTS] Clear message for Spark ML
dependent tests
### What changes were proposed in this pull request?
Similar to https://github.com/apache/spark/pull/42955, this PR proposes to
correct the message for Spark ML only tests from Spark Connect.
### Why are the changes needed?
Among Spark ML dependent tests, there are some edge tests that can only be
tested using Spark ML features. We need to be clearer about why these cannot be
tested.
### Does this PR introduce _any_ user-facing change?
No, it's test-only.
### How was this patch tested?
Updated the existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43051 from itholic/ml_dependent.
Authored-by: Haejoon Lee <haejoon....@databricks.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py | 4 ++--
.../tests/connect/plot/test_parity_frame_plot_matplotlib.py | 4 ++--
.../pandas/tests/connect/plot/test_parity_frame_plot_plotly.py | 6 +++---
.../tests/connect/plot/test_parity_series_plot_matplotlib.py | 8 ++++----
.../pandas/tests/connect/plot/test_parity_series_plot_plotly.py | 4 ++--
python/pyspark/pandas/tests/connect/test_parity_default_index.py | 4 +---
6 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py
index 24392eaf27c..10054f58501 100644
--- a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py
+++ b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py
@@ -24,11 +24,11 @@ from pyspark.testing.pandasutils import
PandasOnSparkTestUtils
class DataFramePlotParityTests(
DataFramePlotTestsMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_compute_hist_multi_columns(self):
super().test_compute_hist_multi_columns()
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_compute_hist_single_column(self):
super().test_compute_hist_single_column()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
index 3f615326f2b..9fec1c57c02 100644
---
a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
+++
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py
@@ -24,11 +24,11 @@ from pyspark.testing.pandasutils import
PandasOnSparkTestUtils, TestUtils
class DataFramePlotMatplotlibParityTests(
DataFramePlotMatplotlibTestsMixin, PandasOnSparkTestUtils, TestUtils,
ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
index 16b97d6814e..452962d8135 100644
--- a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
+++ b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py
@@ -24,15 +24,15 @@ from pyspark.testing.pandasutils import
PandasOnSparkTestUtils, TestUtils
class DataFramePlotPlotlyParityTests(
DataFramePlotPlotlyTestsMixin, PandasOnSparkTestUtils, TestUtils,
ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_hist_layout_kwargs(self):
super().test_hist_layout_kwargs()
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
index 9e264e76229..f093f48b16e 100644
---
a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
+++
b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py
@@ -24,19 +24,19 @@ from pyspark.testing.pandasutils import
PandasOnSparkTestUtils, TestUtils
class SeriesPlotMatplotlibParityTests(
SeriesPlotMatplotlibTestsMixin, PandasOnSparkTestUtils, TestUtils,
ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_hist(self):
super().test_hist()
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_single_value_hist(self):
super().test_single_value_hist()
diff --git
a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
index 7d6c92e56ac..795732950b8 100644
--- a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
+++ b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py
@@ -24,11 +24,11 @@ from pyspark.testing.pandasutils import
PandasOnSparkTestUtils, TestUtils
class SeriesPlotPlotlyParityTests(
SeriesPlotPlotlyTestsMixin, PandasOnSparkTestUtils, TestUtils,
ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with
Spark Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_hist_plot(self):
super().test_hist_plot()
- @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark
Connect.")
+ @unittest.skip("Test depends on Spark ML which is not supported from Spark
Connect.")
def test_kde_plot(self):
super().test_kde_plot()
diff --git a/python/pyspark/pandas/tests/connect/test_parity_default_index.py
b/python/pyspark/pandas/tests/connect/test_parity_default_index.py
index 1e95fac9285..a249fd7ef91 100644
--- a/python/pyspark/pandas/tests/connect/test_parity_default_index.py
+++ b/python/pyspark/pandas/tests/connect/test_parity_default_index.py
@@ -24,9 +24,7 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class DefaultIndexParityTests(
DefaultIndexTestsMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
- @unittest.skip(
- "TODO(SPARK-43623): Enable
DefaultIndexParityTests.test_index_distributed_sequence_cleanup."
- )
+ @unittest.skip("Test depends on SparkContext which is not supported from
Spark Connect.")
def test_index_distributed_sequence_cleanup(self):
super().test_index_distributed_sequence_cleanup()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
