This is an automated email from the ASF dual-hosted git repository. ruifengz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new ee0d243f0141 [SPARK-46719][PS][TESTS] Rebalance `pyspark_pandas` and `pyspark_pandas_slow` ee0d243f0141 is described below commit ee0d243f014176ede9d8cf8296f9a2df1798920b Author: Ruifeng Zheng <ruife...@apache.org> AuthorDate: Mon Jan 15 15:10:55 2024 +0800 [SPARK-46719][PS][TESTS] Rebalance `pyspark_pandas` and `pyspark_pandas_slow` ### What changes were proposed in this pull request? Rebalance `pyspark_pandas` and `pyspark_pandas_slow` ### Why are the changes needed? before: `pyspark_pandas`: `Tests passed in 1849 seconds` `pyspark_pandas-slow`: `Tests passed in 3538 seconds` after: `pyspark_pandas`: `Tests passed in 2733 seconds` `pyspark_pandas-slow`: `Tests passed in 2804 seconds` ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? ci, https://github.com/zhengruifeng/spark/actions/runs/7524159324/job/20478674209 ### Was this patch authored or co-authored using generative AI tooling? no Closes #44731 from zhengruifeng/infra_rebalance_ps_test. Authored-by: Ruifeng Zheng <ruife...@apache.org> Signed-off-by: Ruifeng Zheng <ruife...@apache.org> --- dev/sparktestsupport/modules.py | 158 ++++++++++++++++++++-------------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 9cdbe4d250ca..202263febc93 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -699,6 +699,40 @@ pyspark_pandas = Module( "pyspark.pandas.spark.utils", "pyspark.pandas.typedef.typehints", # unittests + "pyspark.pandas.tests.test_categorical", + "pyspark.pandas.tests.test_config", + "pyspark.pandas.tests.test_extension", + "pyspark.pandas.tests.test_frame_spark", + "pyspark.pandas.tests.test_generic_functions", + "pyspark.pandas.tests.test_indexops_spark", + "pyspark.pandas.tests.test_internal", + "pyspark.pandas.tests.test_namespace", + "pyspark.pandas.tests.test_numpy_compat", + "pyspark.pandas.tests.test_repr", + "pyspark.pandas.tests.test_spark_functions", + "pyspark.pandas.tests.test_scalars", + "pyspark.pandas.tests.test_sql", + "pyspark.pandas.tests.test_typedef", + "pyspark.pandas.tests.test_utils", + "pyspark.pandas.tests.computation.test_any_all", + "pyspark.pandas.tests.computation.test_apply_func", + "pyspark.pandas.tests.computation.test_binary_ops", + "pyspark.pandas.tests.computation.test_combine", + "pyspark.pandas.tests.computation.test_compute", + "pyspark.pandas.tests.computation.test_corr", + "pyspark.pandas.tests.computation.test_corrwith", + "pyspark.pandas.tests.computation.test_cov", + "pyspark.pandas.tests.computation.test_cumulative", + "pyspark.pandas.tests.computation.test_describe", + "pyspark.pandas.tests.computation.test_eval", + "pyspark.pandas.tests.computation.test_melt", + "pyspark.pandas.tests.computation.test_missing_data", + "pyspark.pandas.tests.computation.test_pivot", + "pyspark.pandas.tests.computation.test_pivot_table", + "pyspark.pandas.tests.computation.test_pivot_table_adv", + "pyspark.pandas.tests.computation.test_pivot_table_multi_idx", + "pyspark.pandas.tests.computation.test_pivot_table_multi_idx_adv", + "pyspark.pandas.tests.computation.test_stats", "pyspark.pandas.tests.data_type_ops.test_as_type", "pyspark.pandas.tests.data_type_ops.test_base", "pyspark.pandas.tests.data_type_ops.test_binary_ops", @@ -717,42 +751,26 @@ pyspark_pandas = Module( "pyspark.pandas.tests.data_type_ops.test_string_ops", "pyspark.pandas.tests.data_type_ops.test_udt_ops", "pyspark.pandas.tests.data_type_ops.test_timedelta_ops", - "pyspark.pandas.tests.indexes.test_category", - "pyspark.pandas.tests.indexes.test_timedelta", "pyspark.pandas.tests.plot.test_frame_plot", "pyspark.pandas.tests.plot.test_frame_plot_matplotlib", "pyspark.pandas.tests.plot.test_frame_plot_plotly", "pyspark.pandas.tests.plot.test_series_plot", "pyspark.pandas.tests.plot.test_series_plot_matplotlib", "pyspark.pandas.tests.plot.test_series_plot_plotly", - "pyspark.pandas.tests.test_categorical", - "pyspark.pandas.tests.test_config", - "pyspark.pandas.tests.indexes.test_default", - "pyspark.pandas.tests.window.test_expanding", - "pyspark.pandas.tests.window.test_expanding_adv", - "pyspark.pandas.tests.window.test_expanding_error", - "pyspark.pandas.tests.window.test_groupby_expanding", - "pyspark.pandas.tests.window.test_groupby_expanding_adv", - "pyspark.pandas.tests.test_extension", - "pyspark.pandas.tests.window.test_ewm_error", - "pyspark.pandas.tests.window.test_ewm_mean", - "pyspark.pandas.tests.window.test_groupby_ewm_mean", - "pyspark.pandas.tests.test_frame_spark", - "pyspark.pandas.tests.test_generic_functions", "pyspark.pandas.tests.frame.test_interpolate", "pyspark.pandas.tests.frame.test_interpolate_error", + "pyspark.pandas.tests.frame.test_attrs", + "pyspark.pandas.tests.frame.test_axis", + "pyspark.pandas.tests.frame.test_constructor", + "pyspark.pandas.tests.frame.test_conversion", + "pyspark.pandas.tests.frame.test_reindexing", + "pyspark.pandas.tests.frame.test_reshaping", + "pyspark.pandas.tests.frame.test_spark", + "pyspark.pandas.tests.frame.test_take", + "pyspark.pandas.tests.frame.test_take_adv", + "pyspark.pandas.tests.frame.test_time_series", + "pyspark.pandas.tests.frame.test_truncate", "pyspark.pandas.tests.series.test_interpolate", - "pyspark.pandas.tests.test_indexops_spark", - "pyspark.pandas.tests.test_internal", - "pyspark.pandas.tests.test_namespace", - "pyspark.pandas.tests.test_numpy_compat", - "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding", - "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding_adv", - "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding_count", - "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling", - "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_adv", - "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_count", - "pyspark.pandas.tests.test_repr", "pyspark.pandas.tests.resample.test_on", "pyspark.pandas.tests.resample.test_error", "pyspark.pandas.tests.resample.test_frame", @@ -765,6 +783,14 @@ pyspark_pandas = Module( "pyspark.pandas.tests.reshape.test_get_dummies_object", "pyspark.pandas.tests.reshape.test_get_dummies_prefix", "pyspark.pandas.tests.reshape.test_merge_asof", + "pyspark.pandas.tests.window.test_expanding", + "pyspark.pandas.tests.window.test_expanding_adv", + "pyspark.pandas.tests.window.test_expanding_error", + "pyspark.pandas.tests.window.test_groupby_expanding", + "pyspark.pandas.tests.window.test_groupby_expanding_adv", + "pyspark.pandas.tests.window.test_ewm_error", + "pyspark.pandas.tests.window.test_ewm_mean", + "pyspark.pandas.tests.window.test_groupby_ewm_mean", "pyspark.pandas.tests.window.test_missing", "pyspark.pandas.tests.window.test_rolling", "pyspark.pandas.tests.window.test_rolling_adv", @@ -773,14 +799,26 @@ pyspark_pandas = Module( "pyspark.pandas.tests.window.test_groupby_rolling", "pyspark.pandas.tests.window.test_groupby_rolling_adv", "pyspark.pandas.tests.window.test_groupby_rolling_count", - "pyspark.pandas.tests.test_scalars", "pyspark.pandas.tests.series.test_datetime", "pyspark.pandas.tests.series.test_string_ops_adv", "pyspark.pandas.tests.series.test_string_ops_basic", - "pyspark.pandas.tests.test_spark_functions", - "pyspark.pandas.tests.test_sql", - "pyspark.pandas.tests.test_typedef", - "pyspark.pandas.tests.test_utils", + "pyspark.pandas.tests.series.test_all_any", + "pyspark.pandas.tests.series.test_arg_ops", + "pyspark.pandas.tests.series.test_as_of", + "pyspark.pandas.tests.series.test_as_type", + "pyspark.pandas.tests.series.test_compute", + "pyspark.pandas.tests.series.test_conversion", + "pyspark.pandas.tests.series.test_cumulative", + "pyspark.pandas.tests.series.test_index", + "pyspark.pandas.tests.series.test_missing_data", + "pyspark.pandas.tests.series.test_series", + "pyspark.pandas.tests.series.test_sort", + "pyspark.pandas.tests.series.test_stat", + "pyspark.pandas.tests.io.test_io", + "pyspark.pandas.tests.io.test_csv", + "pyspark.pandas.tests.io.test_dataframe_conversion", + "pyspark.pandas.tests.io.test_dataframe_spark_io", + "pyspark.pandas.tests.io.test_series_conversion", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and @@ -798,6 +836,9 @@ pyspark_pandas_slow = Module( "pyspark.pandas.generic", "pyspark.pandas.series", # unittests + "pyspark.pandas.tests.indexes.test_default", + "pyspark.pandas.tests.indexes.test_category", + "pyspark.pandas.tests.indexes.test_timedelta", "pyspark.pandas.tests.indexes.test_basic", "pyspark.pandas.tests.indexes.test_getattr", "pyspark.pandas.tests.indexes.test_name", @@ -841,41 +882,6 @@ pyspark_pandas_slow = Module( "pyspark.pandas.tests.indexes.test_reindex", "pyspark.pandas.tests.indexes.test_rename", "pyspark.pandas.tests.indexes.test_reset_index", - "pyspark.pandas.tests.computation.test_any_all", - "pyspark.pandas.tests.computation.test_apply_func", - "pyspark.pandas.tests.computation.test_binary_ops", - "pyspark.pandas.tests.computation.test_combine", - "pyspark.pandas.tests.computation.test_compute", - "pyspark.pandas.tests.computation.test_corr", - "pyspark.pandas.tests.computation.test_corrwith", - "pyspark.pandas.tests.computation.test_cov", - "pyspark.pandas.tests.computation.test_cumulative", - "pyspark.pandas.tests.computation.test_describe", - "pyspark.pandas.tests.computation.test_eval", - "pyspark.pandas.tests.computation.test_melt", - "pyspark.pandas.tests.computation.test_missing_data", - "pyspark.pandas.tests.computation.test_pivot", - "pyspark.pandas.tests.computation.test_pivot_table", - "pyspark.pandas.tests.computation.test_pivot_table_adv", - "pyspark.pandas.tests.computation.test_pivot_table_multi_idx", - "pyspark.pandas.tests.computation.test_pivot_table_multi_idx_adv", - "pyspark.pandas.tests.computation.test_stats", - "pyspark.pandas.tests.frame.test_attrs", - "pyspark.pandas.tests.frame.test_axis", - "pyspark.pandas.tests.frame.test_constructor", - "pyspark.pandas.tests.frame.test_conversion", - "pyspark.pandas.tests.frame.test_reindexing", - "pyspark.pandas.tests.frame.test_reshaping", - "pyspark.pandas.tests.frame.test_spark", - "pyspark.pandas.tests.frame.test_take", - "pyspark.pandas.tests.frame.test_take_adv", - "pyspark.pandas.tests.frame.test_time_series", - "pyspark.pandas.tests.frame.test_truncate", - "pyspark.pandas.tests.io.test_io", - "pyspark.pandas.tests.io.test_csv", - "pyspark.pandas.tests.io.test_dataframe_conversion", - "pyspark.pandas.tests.io.test_dataframe_spark_io", - "pyspark.pandas.tests.io.test_series_conversion", "pyspark.pandas.tests.groupby.test_aggregate", "pyspark.pandas.tests.groupby.test_apply_func", "pyspark.pandas.tests.groupby.test_cumulative", @@ -936,18 +942,12 @@ pyspark_pandas_slow = Module( "pyspark.pandas.tests.diff_frames_ops.test_groupby_shift", "pyspark.pandas.tests.diff_frames_ops.test_groupby_split_apply_combine", "pyspark.pandas.tests.diff_frames_ops.test_groupby_transform", - "pyspark.pandas.tests.series.test_all_any", - "pyspark.pandas.tests.series.test_arg_ops", - "pyspark.pandas.tests.series.test_as_of", - "pyspark.pandas.tests.series.test_as_type", - "pyspark.pandas.tests.series.test_compute", - "pyspark.pandas.tests.series.test_conversion", - "pyspark.pandas.tests.series.test_cumulative", - "pyspark.pandas.tests.series.test_index", - "pyspark.pandas.tests.series.test_missing_data", - "pyspark.pandas.tests.series.test_series", - "pyspark.pandas.tests.series.test_sort", - "pyspark.pandas.tests.series.test_stat", + "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding", + "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding_adv", + "pyspark.pandas.tests.diff_frames_ops.test_groupby_expanding_count", + "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling", + "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_adv", + "pyspark.pandas.tests.diff_frames_ops.test_groupby_rolling_count", ], excluded_python_implementations=[ "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org