This is an automated email from the ASF dual-hosted git repository. ruifengz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new be95298f246 [SPARK-44524][BUILD] Balancing pyspark-pandas-connect and pyspark-pandas-slow-connect GA testing time be95298f246 is described below commit be95298f24669cddaa65e412d7681476149e2977 Author: panbingkun <pbk1...@gmail.com> AuthorDate: Mon Jul 31 09:13:18 2023 +0800 [SPARK-44524][BUILD] Balancing pyspark-pandas-connect and pyspark-pandas-slow-connect GA testing time ### What changes were proposed in this pull request? The pr aims to balancing `pyspark-pandas-connect` and `pyspark-pandas-slow-connect` GA testing time. ### Why are the changes needed? After pr: https://github.com/apache/spark/pull/42146, the difference in testing time between `pyspark-pandas-connect` and `pyspark-pandas-slow-connect` is a bit significant, which affects the overall running time. In order to make GA operation more efficient and stable. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? - Pass GA. - Manually monitor GA. Closes #42115 from panbingkun/free_disk_space. Lead-authored-by: panbingkun <pbk1...@gmail.com> Co-authored-by: panbingkun <84731...@qq.com> Signed-off-by: Ruifeng Zheng <ruife...@apache.org> --- dev/sparktestsupport/modules.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 79c3f8f26b1..6830039ba1c 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -891,7 +891,7 @@ pyspark_connect = Module( pyspark_pandas_connect = Module( name="pyspark-pandas-connect", - dependencies=[pyspark_connect, pyspark_pandas], + dependencies=[pyspark_connect, pyspark_pandas, pyspark_pandas_slow], source_file_regexes=[ "python/pyspark/pandas", ], @@ -949,23 +949,6 @@ pyspark_pandas_connect = Module( "pyspark.pandas.tests.connect.test_parity_utils", "pyspark.pandas.tests.connect.test_parity_window", "pyspark.pandas.tests.connect.indexes.test_parity_base", - ], - excluded_python_implementations=[ - "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and - # they aren't available there - ], -) - - -# This module should contain the same test list with 'pyspark_pandas_slow' for maintenance. -pyspark_pandas_slow_connect = Module( - name="pyspark-pandas-slow-connect", - dependencies=[pyspark_connect, pyspark_pandas_slow], - source_file_regexes=[ - "python/pyspark/pandas", - ], - python_test_goals=[ - # pandas-on-Spark unittests "pyspark.pandas.tests.connect.indexes.test_parity_datetime", "pyspark.pandas.tests.connect.indexes.test_parity_align", "pyspark.pandas.tests.connect.indexes.test_parity_indexing", @@ -985,6 +968,22 @@ pyspark_pandas_slow_connect = Module( "pyspark.pandas.tests.connect.computation.test_parity_melt", "pyspark.pandas.tests.connect.computation.test_parity_missing_data", "pyspark.pandas.tests.connect.computation.test_parity_pivot", + ], + excluded_python_implementations=[ + "PyPy" # Skip these tests under PyPy since they require numpy, pandas, and pyarrow and + # they aren't available there + ], +) + + +pyspark_pandas_slow_connect = Module( + name="pyspark-pandas-slow-connect", + dependencies=[pyspark_connect, pyspark_pandas, pyspark_pandas_slow], + source_file_regexes=[ + "python/pyspark/pandas", + ], + python_test_goals=[ + # pandas-on-Spark unittests "pyspark.pandas.tests.connect.frame.test_parity_attrs", "pyspark.pandas.tests.connect.frame.test_parity_constructor", "pyspark.pandas.tests.connect.frame.test_parity_conversion", --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org