This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 36e626bc60a [SPARK-43711][SPARK-44372][CONNECT][PS][TESTS] Clear message for Spark ML dependent tests 36e626bc60a is described below commit 36e626bc60af4ce94a1ca304e05390418b965135 Author: Haejoon Lee <haejoon....@databricks.com> AuthorDate: Mon Sep 25 11:23:51 2023 -0700 [SPARK-43711][SPARK-44372][CONNECT][PS][TESTS] Clear message for Spark ML dependent tests ### What changes were proposed in this pull request? Similar to https://github.com/apache/spark/pull/42955, this PR proposes to correct the message for Spark ML only tests from Spark Connect. ### Why are the changes needed? Among Spark ML dependent tests, there are some edge tests that can only be tested using Spark ML features. We need to be clearer about why these cannot be tested. ### Does this PR introduce _any_ user-facing change? No, it's test-only. ### How was this patch tested? Updated the existing tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43051 from itholic/ml_dependent. Authored-by: Haejoon Lee <haejoon....@databricks.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py | 4 ++-- .../tests/connect/plot/test_parity_frame_plot_matplotlib.py | 4 ++-- .../pandas/tests/connect/plot/test_parity_frame_plot_plotly.py | 6 +++--- .../tests/connect/plot/test_parity_series_plot_matplotlib.py | 8 ++++---- .../pandas/tests/connect/plot/test_parity_series_plot_plotly.py | 4 ++-- python/pyspark/pandas/tests/connect/test_parity_default_index.py | 4 +--- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py index 24392eaf27c..10054f58501 100644 --- a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py +++ b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py @@ -24,11 +24,11 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils class DataFramePlotParityTests( DataFramePlotTestsMixin, PandasOnSparkTestUtils, ReusedConnectTestCase ): - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_compute_hist_multi_columns(self): super().test_compute_hist_multi_columns() - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_compute_hist_single_column(self): super().test_compute_hist_single_column() diff --git a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py index 3f615326f2b..9fec1c57c02 100644 --- a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py +++ b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py @@ -24,11 +24,11 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils, TestUtils class DataFramePlotMatplotlibParityTests( DataFramePlotMatplotlibTestsMixin, PandasOnSparkTestUtils, TestUtils, ReusedConnectTestCase ): - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_hist_plot(self): super().test_hist_plot() - @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_kde_plot(self): super().test_kde_plot() diff --git a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py index 16b97d6814e..452962d8135 100644 --- a/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py +++ b/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py @@ -24,15 +24,15 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils, TestUtils class DataFramePlotPlotlyParityTests( DataFramePlotPlotlyTestsMixin, PandasOnSparkTestUtils, TestUtils, ReusedConnectTestCase ): - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_hist_layout_kwargs(self): super().test_hist_layout_kwargs() - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_hist_plot(self): super().test_hist_plot() - @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_kde_plot(self): super().test_kde_plot() diff --git a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py index 9e264e76229..f093f48b16e 100644 --- a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py +++ b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py @@ -24,19 +24,19 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils, TestUtils class SeriesPlotMatplotlibParityTests( SeriesPlotMatplotlibTestsMixin, PandasOnSparkTestUtils, TestUtils, ReusedConnectTestCase ): - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_hist(self): super().test_hist() - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_hist_plot(self): super().test_hist_plot() - @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_kde_plot(self): super().test_kde_plot() - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_single_value_hist(self): super().test_single_value_hist() diff --git a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py index 7d6c92e56ac..795732950b8 100644 --- a/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py +++ b/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py @@ -24,11 +24,11 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils, TestUtils class SeriesPlotPlotlyParityTests( SeriesPlotPlotlyTestsMixin, PandasOnSparkTestUtils, TestUtils, ReusedConnectTestCase ): - @unittest.skip("TODO(SPARK-43711): Fix Transformer.transform to work with Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_hist_plot(self): super().test_hist_plot() - @unittest.skip("TODO(SPARK-44372): Enable KernelDensity within Spark Connect.") + @unittest.skip("Test depends on Spark ML which is not supported from Spark Connect.") def test_kde_plot(self): super().test_kde_plot() diff --git a/python/pyspark/pandas/tests/connect/test_parity_default_index.py b/python/pyspark/pandas/tests/connect/test_parity_default_index.py index 1e95fac9285..a249fd7ef91 100644 --- a/python/pyspark/pandas/tests/connect/test_parity_default_index.py +++ b/python/pyspark/pandas/tests/connect/test_parity_default_index.py @@ -24,9 +24,7 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils class DefaultIndexParityTests( DefaultIndexTestsMixin, PandasOnSparkTestUtils, ReusedConnectTestCase ): - @unittest.skip( - "TODO(SPARK-43623): Enable DefaultIndexParityTests.test_index_distributed_sequence_cleanup." - ) + @unittest.skip("Test depends on SparkContext which is not supported from Spark Connect.") def test_index_distributed_sequence_cleanup(self): super().test_index_distributed_sequence_cleanup() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org