This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new bf4981fd4ad [SPARK-41457][PYTHON][TESTS] Refactor type annotations and
dependency checks in tests
bf4981fd4ad is described below
commit bf4981fd4adfe96d3962e2e165c5a5d307a0033d
Author: Hyukjin Kwon <gurwls...@apache.org>
AuthorDate: Fri Dec 9 21:34:57 2022 -0800
[SPARK-41457][PYTHON][TESTS] Refactor type annotations and dependency
checks in tests
### What changes were proposed in this pull request?
This PR proposes to:
- Print out the correct error message when dependencies are not installed
for `pyspark.sql.connect` in main code.
- Print out the correct message if 1. dependencies are missing or 2. build
is not there in Spark Connect in tests.
- Ignore all MyPy errors from tests:
- Initial intention was to annotate types for public APIs only.
- Also removed unused ignores in other modules
Note that, for Spark Connect, we should follow both below to make sure they
are properly skipped when dependences are not there, or jars do not exist.
- Always use `if should_test_connect` for `pyspark.sql.connect` and related
dependency imports in tests.
- Use `unittest.skipIf(not should_test_connect,
connect_requirement_message)`
### Why are the changes needed?
- To make the development easier and faster.
- To make sure our Jenkins build pass without Python dependencies installed
- See [Scaleaway](https://apache-spark.s3.fr-par.scw.cloud/index.html),
see also
https://github.com/apache/spark-website/blob/asf-site/developer-tools.md#test-coverage.
### Does this PR introduce _any_ user-facing change?
No, dev and test-only.
### How was this patch tested?
Manually tested without pandas, pyarrow and grpc, and verified the error
messages.
Closes #38991 from HyukjinKwon/SPARK-41457.
Authored-by: Hyukjin Kwon <gurwls...@apache.org>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
dev/tox.ini | 3 +-
python/mypy.ini | 53 ++++++------
python/pyspark/ml/tests/test_algorithms.py | 2 +-
python/pyspark/ml/tests/test_base.py | 2 +-
python/pyspark/ml/tests/test_evaluation.py | 2 +-
python/pyspark/ml/tests/test_feature.py | 2 +-
python/pyspark/ml/tests/test_image.py | 2 +-
python/pyspark/ml/tests/test_linalg.py | 2 +-
python/pyspark/ml/tests/test_param.py | 2 +-
python/pyspark/ml/tests/test_persistence.py | 2 +-
python/pyspark/ml/tests/test_pipeline.py | 2 +-
python/pyspark/ml/tests/test_stat.py | 2 +-
python/pyspark/ml/tests/test_training_summary.py | 2 +-
python/pyspark/ml/tests/test_tuning.py | 2 +-
python/pyspark/ml/tests/test_util.py | 2 +-
python/pyspark/ml/tests/test_wrapper.py | 2 +-
python/pyspark/mllib/tests/test_algorithms.py | 2 +-
python/pyspark/mllib/tests/test_feature.py | 2 +-
python/pyspark/mllib/tests/test_linalg.py | 2 +-
python/pyspark/mllib/tests/test_stat.py | 2 +-
.../mllib/tests/test_streaming_algorithms.py | 2 +-
python/pyspark/mllib/tests/test_util.py | 2 +-
.../pandas/tests/data_type_ops/test_base.py | 2 +-
.../pandas/tests/data_type_ops/test_binary_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_boolean_ops.py | 2 +-
.../tests/data_type_ops/test_categorical_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_complex_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_date_ops.py | 2 +-
.../tests/data_type_ops/test_datetime_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_null_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_num_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_string_ops.py | 2 +-
.../tests/data_type_ops/test_timedelta_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_udt_ops.py | 2 +-
python/pyspark/pandas/tests/indexes/test_base.py | 2 +-
.../pyspark/pandas/tests/indexes/test_category.py | 2 +-
.../pyspark/pandas/tests/indexes/test_datetime.py | 2 +-
.../pyspark/pandas/tests/indexes/test_timedelta.py | 2 +-
.../pyspark/pandas/tests/plot/test_frame_plot.py | 2 +-
.../tests/plot/test_frame_plot_matplotlib.py | 2 +-
.../pandas/tests/plot/test_frame_plot_plotly.py | 2 +-
.../pyspark/pandas/tests/plot/test_series_plot.py | 2 +-
.../tests/plot/test_series_plot_matplotlib.py | 2 +-
.../pandas/tests/plot/test_series_plot_plotly.py | 2 +-
python/pyspark/pandas/tests/test_categorical.py | 4 +-
python/pyspark/pandas/tests/test_config.py | 2 +-
python/pyspark/pandas/tests/test_csv.py | 2 +-
python/pyspark/pandas/tests/test_dataframe.py | 2 +-
.../pandas/tests/test_dataframe_conversion.py | 2 +-
.../pandas/tests/test_dataframe_spark_io.py | 2 +-
python/pyspark/pandas/tests/test_default_index.py | 2 +-
python/pyspark/pandas/tests/test_ewm.py | 2 +-
python/pyspark/pandas/tests/test_expanding.py | 2 +-
python/pyspark/pandas/tests/test_extension.py | 2 +-
python/pyspark/pandas/tests/test_frame_spark.py | 2 +-
.../pyspark/pandas/tests/test_generic_functions.py | 2 +-
python/pyspark/pandas/tests/test_groupby.py | 4 +-
python/pyspark/pandas/tests/test_indexing.py | 2 +-
python/pyspark/pandas/tests/test_indexops_spark.py | 2 +-
python/pyspark/pandas/tests/test_internal.py | 2 +-
python/pyspark/pandas/tests/test_namespace.py | 2 +-
python/pyspark/pandas/tests/test_numpy_compat.py | 2 +-
.../pandas/tests/test_ops_on_diff_frames.py | 2 +-
.../tests/test_ops_on_diff_frames_groupby.py | 2 +-
.../test_ops_on_diff_frames_groupby_expanding.py | 2 +-
.../test_ops_on_diff_frames_groupby_rolling.py | 2 +-
python/pyspark/pandas/tests/test_repr.py | 2 +-
python/pyspark/pandas/tests/test_resample.py | 2 +-
python/pyspark/pandas/tests/test_reshape.py | 2 +-
python/pyspark/pandas/tests/test_rolling.py | 2 +-
python/pyspark/pandas/tests/test_scalars.py | 2 +-
python/pyspark/pandas/tests/test_series.py | 2 +-
.../pyspark/pandas/tests/test_series_conversion.py | 2 +-
.../pyspark/pandas/tests/test_series_datetime.py | 2 +-
python/pyspark/pandas/tests/test_series_string.py | 2 +-
.../pyspark/pandas/tests/test_spark_functions.py | 2 +-
python/pyspark/pandas/tests/test_sql.py | 2 +-
python/pyspark/pandas/tests/test_stats.py | 2 +-
python/pyspark/pandas/tests/test_typedef.py | 16 ++--
python/pyspark/pandas/tests/test_utils.py | 2 +-
python/pyspark/pandas/tests/test_window.py | 2 +-
python/pyspark/resource/tests/test_resources.py | 2 +-
python/pyspark/sql/connect/__init__.py | 10 ++-
python/pyspark/sql/connect/client.py | 2 +-
python/pyspark/sql/pandas/utils.py | 19 +++++
.../sql/tests/connect/test_connect_basic.py | 47 ++++-------
.../sql/tests/connect/test_connect_column.py | 17 ++--
.../connect/test_connect_column_expressions.py | 14 ++--
.../sql/tests/connect/test_connect_function.py | 28 ++-----
.../sql/tests/connect/test_connect_plan_only.py | 14 ++--
.../sql/tests/connect/test_connect_select_ops.py | 12 +--
.../sql/tests/pandas/test_pandas_cogrouped_map.py | 2 +-
.../sql/tests/pandas/test_pandas_grouped_map.py | 2 +-
.../pandas/test_pandas_grouped_map_with_state.py | 2 +-
python/pyspark/sql/tests/pandas/test_pandas_map.py | 2 +-
python/pyspark/sql/tests/pandas/test_pandas_udf.py | 2 +-
.../tests/pandas/test_pandas_udf_grouped_agg.py | 2 +-
.../sql/tests/pandas/test_pandas_udf_scalar.py | 2 +-
.../sql/tests/pandas/test_pandas_udf_typehints.py | 4 +-
...pandas_udf_typehints_with_future_annotations.py | 4 +-
.../sql/tests/pandas/test_pandas_udf_window.py | 2 +-
.../pyspark/sql/tests/streaming/test_streaming.py | 2 +-
.../sql/tests/streaming/test_streaming_listener.py | 2 +-
python/pyspark/sql/tests/test_arrow_map.py | 4 +-
python/pyspark/sql/tests/test_catalog.py | 2 +-
python/pyspark/sql/tests/test_column.py | 2 +-
python/pyspark/sql/tests/test_conf.py | 2 +-
python/pyspark/sql/tests/test_context.py | 2 +-
python/pyspark/sql/tests/test_datasources.py | 2 +-
python/pyspark/sql/tests/test_functions.py | 2 +-
python/pyspark/sql/tests/test_group.py | 2 +-
python/pyspark/sql/tests/test_pandas_sqlmetrics.py | 2 +-
python/pyspark/sql/tests/test_readwriter.py | 2 +-
python/pyspark/sql/tests/test_serde.py | 2 +-
python/pyspark/sql/tests/test_session.py | 2 +-
python/pyspark/sql/tests/test_types.py | 2 +-
python/pyspark/sql/tests/test_utils.py | 2 +-
python/pyspark/streaming/tests/test_context.py | 2 +-
python/pyspark/streaming/tests/test_dstream.py | 2 +-
python/pyspark/streaming/tests/test_kinesis.py | 2 +-
python/pyspark/streaming/tests/test_listener.py | 2 +-
python/pyspark/testing/connectutils.py | 93 ++++++++++++----------
python/pyspark/testing/pandasutils.py | 20 +++--
python/pyspark/tests/test_appsubmit.py | 2 +-
python/pyspark/tests/test_broadcast.py | 2 +-
python/pyspark/tests/test_conf.py | 2 +-
python/pyspark/tests/test_context.py | 8 +-
python/pyspark/tests/test_daemon.py | 2 +-
python/pyspark/tests/test_install_spark.py | 2 +-
python/pyspark/tests/test_join.py | 2 +-
python/pyspark/tests/test_memory_profiler.py | 4 +-
python/pyspark/tests/test_pin_thread.py | 2 +-
python/pyspark/tests/test_profiler.py | 2 +-
python/pyspark/tests/test_rdd.py | 2 +-
python/pyspark/tests/test_rddbarrier.py | 2 +-
python/pyspark/tests/test_rddsampler.py | 2 +-
python/pyspark/tests/test_readwrite.py | 2 +-
python/pyspark/tests/test_serializers.py | 2 +-
python/pyspark/tests/test_shuffle.py | 2 +-
python/pyspark/tests/test_statcounter.py | 2 +-
python/pyspark/tests/test_taskcontext.py | 2 +-
python/pyspark/tests/test_util.py | 2 +-
python/pyspark/tests/test_worker.py | 2 +-
143 files changed, 323 insertions(+), 301 deletions(-)
diff --git a/dev/tox.ini b/dev/tox.ini
index f44cbe54ddf..15c93832c2c 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -36,7 +36,8 @@ per-file-ignores =
python/pyspark/resource/tests/*.py: F403,
python/pyspark/sql/tests/*.py: F403,
python/pyspark/streaming/tests/*.py: F403,
- python/pyspark/tests/*.py: F403
+ python/pyspark/tests/*.py: F403,
+ python/pyspark/testing/*: F401
exclude =
*/target/*,
docs/.local_ruby_bundle/,
diff --git a/python/mypy.ini b/python/mypy.ini
index 927254d3b35..603647bd3cd 100644
--- a/python/mypy.ini
+++ b/python/mypy.ini
@@ -26,7 +26,7 @@ warn_redundant_casts = True
[mypy-pyspark.sql.connect.proto.*]
ignore_errors = True
-; Allow untyped def in internal modules and tests
+; Allow untyped def in internal modules
[mypy-pyspark.daemon]
disallow_untyped_defs = False
@@ -46,33 +46,18 @@ disallow_untyped_defs = False
[mypy-pyspark.join]
disallow_untyped_defs = False
-[mypy-pyspark.ml.tests.*]
-disallow_untyped_defs = False
-
-[mypy-pyspark.mllib.tests.*]
-disallow_untyped_defs = False
-
[mypy-pyspark.rddsampler]
disallow_untyped_defs = False
-[mypy-pyspark.resource.tests.*]
-disallow_untyped_defs = False
-
[mypy-pyspark.serializers]
disallow_untyped_defs = False
[mypy-pyspark.shuffle]
disallow_untyped_defs = False
-[mypy-pyspark.streaming.tests.*]
-disallow_untyped_defs = False
-
[mypy-pyspark.streaming.util]
disallow_untyped_defs = False
-[mypy-pyspark.sql.tests.*]
-disallow_untyped_defs = False
-
[mypy-pyspark.sql.pandas.serializers]
disallow_untyped_defs = False
@@ -88,20 +73,37 @@ disallow_untyped_defs = False
[mypy-pyspark.pandas.usage_logging.*]
disallow_untyped_defs = False
-[mypy-pyspark.pandas.tests.*]
+[mypy-pyspark.traceback_utils]
disallow_untyped_defs = False
-[mypy-pyspark.tests.*]
+[mypy-pyspark.worker]
disallow_untyped_defs = False
-[mypy-pyspark.testing.*]
-disallow_untyped_defs = False
+; Ignore errors in tests
-[mypy-pyspark.traceback_utils]
-disallow_untyped_defs = False
+[mypy-pyspark.ml.tests.*]
+ignore_errors = True
-[mypy-pyspark.worker]
-disallow_untyped_defs = False
+[mypy-pyspark.mllib.tests.*]
+ignore_errors = True
+
+[mypy-pyspark.resource.tests.*]
+ignore_errors = True
+
+[mypy-pyspark.streaming.tests.*]
+ignore_errors = True
+
+[mypy-pyspark.sql.tests.*]
+ignore_errors = True
+
+[mypy-pyspark.pandas.tests.*]
+ignore_errors = True
+
+[mypy-pyspark.tests.*]
+ignore_errors = True
+
+[mypy-pyspark.testing.*]
+ignore_errors = True
; Allow non-strict optional for pyspark.pandas
@@ -145,6 +147,9 @@ ignore_missing_imports = True
[mypy-google.protobuf.*]
ignore_missing_imports = True
+[mypy-grpc.*]
+ignore_missing_imports = True
+
; Ignore errors for proto generated code
[mypy-pyspark.sql.connect.proto.*, pyspark.sql.connect.proto]
ignore_errors = True
diff --git a/python/pyspark/ml/tests/test_algorithms.py
b/python/pyspark/ml/tests/test_algorithms.py
index e677e79cecb..accdddb29c0 100644
--- a/python/pyspark/ml/tests/test_algorithms.py
+++ b/python/pyspark/ml/tests/test_algorithms.py
@@ -413,7 +413,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_algorithms import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_base.py
b/python/pyspark/ml/tests/test_base.py
index b95b8fbdd54..6c3c51d1c09 100644
--- a/python/pyspark/ml/tests/test_base.py
+++ b/python/pyspark/ml/tests/test_base.py
@@ -88,7 +88,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_base import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_evaluation.py
b/python/pyspark/ml/tests/test_evaluation.py
index d2fd3696240..3c5ae3fbe7d 100644
--- a/python/pyspark/ml/tests/test_evaluation.py
+++ b/python/pyspark/ml/tests/test_evaluation.py
@@ -69,7 +69,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_evaluation import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_feature.py
b/python/pyspark/ml/tests/test_feature.py
index 6cf31758654..0051d47ae33 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -393,7 +393,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_feature import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_image.py
b/python/pyspark/ml/tests/test_image.py
index 8a155ab56ac..86fa46c3248 100644
--- a/python/pyspark/ml/tests/test_image.py
+++ b/python/pyspark/ml/tests/test_image.py
@@ -74,7 +74,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_image import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_linalg.py
b/python/pyspark/ml/tests/test_linalg.py
index a6e9f4e752e..6632d100ea5 100644
--- a/python/pyspark/ml/tests/test_linalg.py
+++ b/python/pyspark/ml/tests/test_linalg.py
@@ -401,7 +401,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_linalg import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_param.py
b/python/pyspark/ml/tests/test_param.py
index 64ed2f6dbe4..8df50a5963e 100644
--- a/python/pyspark/ml/tests/test_param.py
+++ b/python/pyspark/ml/tests/test_param.py
@@ -433,7 +433,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_param import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_persistence.py
b/python/pyspark/ml/tests/test_persistence.py
index 0b54540f06d..406180d9a63 100644
--- a/python/pyspark/ml/tests/test_persistence.py
+++ b/python/pyspark/ml/tests/test_persistence.py
@@ -538,7 +538,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_persistence import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_pipeline.py
b/python/pyspark/ml/tests/test_pipeline.py
index 1f73fdd3445..afc900cec4c 100644
--- a/python/pyspark/ml/tests/test_pipeline.py
+++ b/python/pyspark/ml/tests/test_pipeline.py
@@ -63,7 +63,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_pipeline import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_stat.py
b/python/pyspark/ml/tests/test_stat.py
index 16ce1bc7daa..6bab41b5675 100644
--- a/python/pyspark/ml/tests/test_stat.py
+++ b/python/pyspark/ml/tests/test_stat.py
@@ -44,7 +44,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_stat import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_training_summary.py
b/python/pyspark/ml/tests/test_training_summary.py
index 27d9c182cf7..5704d718673 100644
--- a/python/pyspark/ml/tests/test_training_summary.py
+++ b/python/pyspark/ml/tests/test_training_summary.py
@@ -486,7 +486,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_training_summary import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_tuning.py
b/python/pyspark/ml/tests/test_tuning.py
index c4273f36d74..d9a5c51fd55 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -1027,7 +1027,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_tuning import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_util.py
b/python/pyspark/ml/tests/test_util.py
index 4d5c6a47275..55c973831b5 100644
--- a/python/pyspark/ml/tests/test_util.py
+++ b/python/pyspark/ml/tests/test_util.py
@@ -77,7 +77,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_util import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/ml/tests/test_wrapper.py
b/python/pyspark/ml/tests/test_wrapper.py
index 02ce6f31924..33d93c02acd 100644
--- a/python/pyspark/ml/tests/test_wrapper.py
+++ b/python/pyspark/ml/tests/test_wrapper.py
@@ -130,7 +130,7 @@ if __name__ == "__main__":
from pyspark.ml.tests.test_wrapper import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/mllib/tests/test_algorithms.py
b/python/pyspark/mllib/tests/test_algorithms.py
index 8882242259a..6a9be99ecdf 100644
--- a/python/pyspark/mllib/tests/test_algorithms.py
+++ b/python/pyspark/mllib/tests/test_algorithms.py
@@ -338,7 +338,7 @@ if __name__ == "__main__":
from pyspark.mllib.tests.test_algorithms import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/mllib/tests/test_feature.py
b/python/pyspark/mllib/tests/test_feature.py
index 080a2bf1f59..ca06f39da23 100644
--- a/python/pyspark/mllib/tests/test_feature.py
+++ b/python/pyspark/mllib/tests/test_feature.py
@@ -184,7 +184,7 @@ if __name__ == "__main__":
from pyspark.mllib.tests.test_feature import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/mllib/tests/test_linalg.py
b/python/pyspark/mllib/tests/test_linalg.py
index 007f42d3c2d..d137c888360 100644
--- a/python/pyspark/mllib/tests/test_linalg.py
+++ b/python/pyspark/mllib/tests/test_linalg.py
@@ -672,7 +672,7 @@ if __name__ == "__main__":
from pyspark.mllib.tests.test_linalg import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/mllib/tests/test_stat.py
b/python/pyspark/mllib/tests/test_stat.py
index 7a33d773d15..cef1294ada1 100644
--- a/python/pyspark/mllib/tests/test_stat.py
+++ b/python/pyspark/mllib/tests/test_stat.py
@@ -198,7 +198,7 @@ if __name__ == "__main__":
from pyspark.mllib.tests.test_stat import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py
b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 779fff70902..5a06742ba75 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -463,7 +463,7 @@ if __name__ == "__main__":
from pyspark.mllib.tests.test_streaming_algorithms import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/mllib/tests/test_util.py
b/python/pyspark/mllib/tests/test_util.py
index aad1349c71b..28a53af0aa8 100644
--- a/python/pyspark/mllib/tests/test_util.py
+++ b/python/pyspark/mllib/tests/test_util.py
@@ -100,7 +100,7 @@ if __name__ == "__main__":
from pyspark.mllib.tests.test_util import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_base.py
b/python/pyspark/pandas/tests/data_type_ops/test_base.py
index db4724b9828..9b40d15db6c 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_base.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_base.py
@@ -95,7 +95,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_base import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py
index 7135800bd9b..6eca20d2dbd 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py
@@ -212,7 +212,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_binary_ops import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
index 73761202267..ad7ead6316a 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
@@ -813,7 +813,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_boolean_ops import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py
index 992e3ed70fb..41e6c4885d3 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py
@@ -550,7 +550,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_categorical_ops import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py
index bbdf837ce29..2b85e7bb269 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py
@@ -356,7 +356,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_complex_ops import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
index b457ab2cc85..2fe8a4c688d 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py
@@ -235,7 +235,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_date_ops import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py
index de9c6acb2cd..55d06c07cdd 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py
@@ -248,7 +248,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_datetime_ops import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py
index fc6cdd1a436..44ea159f2a9 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py
@@ -165,7 +165,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_null_ops import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
index cb678ff585e..22d4e8d8ff7 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
@@ -694,7 +694,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_num_ops import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
index cc448dc42d6..cf785f1ebb6 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
@@ -342,7 +342,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_string_ops import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py
index eeaba4d2778..3889520ad8c 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py
@@ -207,7 +207,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_timedelta_ops import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py
b/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py
index 81767af76f9..beebc1f320e 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py
@@ -180,7 +180,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.data_type_ops.test_udt_ops import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/indexes/test_base.py
b/python/pyspark/pandas/tests/indexes/test_base.py
index 9ca31923d5c..0e2c6409796 100644
--- a/python/pyspark/pandas/tests/indexes/test_base.py
+++ b/python/pyspark/pandas/tests/indexes/test_base.py
@@ -2590,7 +2590,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.indexes.test_base import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/indexes/test_category.py
b/python/pyspark/pandas/tests/indexes/test_category.py
index ba737eb520f..10c822a3ca5 100644
--- a/python/pyspark/pandas/tests/indexes/test_category.py
+++ b/python/pyspark/pandas/tests/indexes/test_category.py
@@ -459,7 +459,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.indexes.test_category import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/indexes/test_datetime.py
b/python/pyspark/pandas/tests/indexes/test_datetime.py
index f7155187436..8f8e283f3ab 100644
--- a/python/pyspark/pandas/tests/indexes/test_datetime.py
+++ b/python/pyspark/pandas/tests/indexes/test_datetime.py
@@ -254,7 +254,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.indexes.test_datetime import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/indexes/test_timedelta.py
b/python/pyspark/pandas/tests/indexes/test_timedelta.py
index b191ff8bfbb..654f5ee3a01 100644
--- a/python/pyspark/pandas/tests/indexes/test_timedelta.py
+++ b/python/pyspark/pandas/tests/indexes/test_timedelta.py
@@ -110,7 +110,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.indexes.test_timedelta import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/plot/test_frame_plot.py
b/python/pyspark/pandas/tests/plot/test_frame_plot.py
index 5d265ff2eee..817ea896e79 100644
--- a/python/pyspark/pandas/tests/plot/test_frame_plot.py
+++ b/python/pyspark/pandas/tests/plot/test_frame_plot.py
@@ -158,7 +158,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.plot.test_frame_plot import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py
b/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py
index bb400996e27..7c633710983 100644
--- a/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py
+++ b/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py
@@ -477,7 +477,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.plot.test_frame_plot_matplotlib import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
b/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
index d169326b7b7..f7cf1fc3498 100644
--- a/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
+++ b/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py
@@ -273,7 +273,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.plot.test_frame_plot_plotly import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/plot/test_series_plot.py
b/python/pyspark/pandas/tests/plot/test_series_plot.py
index f3d4ef553bf..fab04bac21d 100644
--- a/python/pyspark/pandas/tests/plot/test_series_plot.py
+++ b/python/pyspark/pandas/tests/plot/test_series_plot.py
@@ -94,7 +94,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.plot.test_series_plot import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
b/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
index 680eee13deb..c17290c44b9 100644
--- a/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
+++ b/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py
@@ -397,7 +397,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.plot.test_series_plot_matplotlib import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
b/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
index 8a50b1829dd..7bd612c1a88 100644
--- a/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
+++ b/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py
@@ -235,7 +235,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.plot.test_series_plot_plotly import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_categorical.py
b/python/pyspark/pandas/tests/test_categorical.py
index 99f315a43a0..d5a660a66ef 100644
--- a/python/pyspark/pandas/tests/test_categorical.py
+++ b/python/pyspark/pandas/tests/test_categorical.py
@@ -436,7 +436,7 @@ class CategoricalTest(ComparisonTestBase, TestUtils):
pdf, psdf = self.df_pair
- def identity(x) -> ps.Series[psdf.b.dtype]: # type:
ignore[name-defined]
+ def identity(x) -> ps.Series[psdf.b.dtype]:
return x
self.assert_eq(
@@ -796,7 +796,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_categorical import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_config.py
b/python/pyspark/pandas/tests/test_config.py
index d3900e216cd..c1c22992407 100644
--- a/python/pyspark/pandas/tests/test_config.py
+++ b/python/pyspark/pandas/tests/test_config.py
@@ -148,7 +148,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_config import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_csv.py
b/python/pyspark/pandas/tests/test_csv.py
index 6bdc989c5da..a94125e6489 100644
--- a/python/pyspark/pandas/tests/test_csv.py
+++ b/python/pyspark/pandas/tests/test_csv.py
@@ -435,7 +435,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_csv import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_dataframe.py
b/python/pyspark/pandas/tests/test_dataframe.py
index ded110c1231..1b06d321e13 100644
--- a/python/pyspark/pandas/tests/test_dataframe.py
+++ b/python/pyspark/pandas/tests/test_dataframe.py
@@ -7106,7 +7106,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_dataframe import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_dataframe_conversion.py
b/python/pyspark/pandas/tests/test_dataframe_conversion.py
index 4e4c9ac2e7d..67ff40e9f15 100644
--- a/python/pyspark/pandas/tests/test_dataframe_conversion.py
+++ b/python/pyspark/pandas/tests/test_dataframe_conversion.py
@@ -262,7 +262,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_dataframe_conversion import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_dataframe_spark_io.py
b/python/pyspark/pandas/tests/test_dataframe_spark_io.py
index dd83070a16c..9904ff032d1 100644
--- a/python/pyspark/pandas/tests/test_dataframe_spark_io.py
+++ b/python/pyspark/pandas/tests/test_dataframe_spark_io.py
@@ -475,7 +475,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_dataframe_spark_io import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_default_index.py
b/python/pyspark/pandas/tests/test_default_index.py
index dcb120aee40..ddd9e296625 100644
--- a/python/pyspark/pandas/tests/test_default_index.py
+++ b/python/pyspark/pandas/tests/test_default_index.py
@@ -97,7 +97,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_default_index import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_ewm.py
b/python/pyspark/pandas/tests/test_ewm.py
index 3ce0bd45071..4d3c98572d8 100644
--- a/python/pyspark/pandas/tests/test_ewm.py
+++ b/python/pyspark/pandas/tests/test_ewm.py
@@ -422,7 +422,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_ewm import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_expanding.py
b/python/pyspark/pandas/tests/test_expanding.py
index 77ced41eb8c..d712f03f7db 100644
--- a/python/pyspark/pandas/tests/test_expanding.py
+++ b/python/pyspark/pandas/tests/test_expanding.py
@@ -241,7 +241,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_expanding import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_extension.py
b/python/pyspark/pandas/tests/test_extension.py
index dd2d08dded0..5d4b5dfa76f 100644
--- a/python/pyspark/pandas/tests/test_extension.py
+++ b/python/pyspark/pandas/tests/test_extension.py
@@ -140,7 +140,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_extension import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_frame_spark.py
b/python/pyspark/pandas/tests/test_frame_spark.py
index 9b47ceca7a4..df090b74d96 100644
--- a/python/pyspark/pandas/tests/test_frame_spark.py
+++ b/python/pyspark/pandas/tests/test_frame_spark.py
@@ -148,7 +148,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_frame_spark import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_generic_functions.py
b/python/pyspark/pandas/tests/test_generic_functions.py
index d4763022059..72e0e47aed0 100644
--- a/python/pyspark/pandas/tests/test_generic_functions.py
+++ b/python/pyspark/pandas/tests/test_generic_functions.py
@@ -222,7 +222,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_generic_functions import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_groupby.py
b/python/pyspark/pandas/tests/test_groupby.py
index a203f77717e..1c940e3abf9 100644
--- a/python/pyspark/pandas/tests/test_groupby.py
+++ b/python/pyspark/pandas/tests/test_groupby.py
@@ -2334,7 +2334,7 @@ class GroupByTest(PandasOnSparkTestCase, TestUtils):
def test_apply_negative(self):
def func(_) -> ps.Series[int]:
- return pd.Series([1]) # type: ignore[return-value]
+ return pd.Series([1])
with self.assertRaisesRegex(TypeError, "Series as a return type hint
at frame groupby"):
ps.range(10).groupby("id").apply(func)
@@ -3242,7 +3242,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_groupby import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_indexing.py
b/python/pyspark/pandas/tests/test_indexing.py
index c939a69929b..9d52c412748 100644
--- a/python/pyspark/pandas/tests/test_indexing.py
+++ b/python/pyspark/pandas/tests/test_indexing.py
@@ -1327,7 +1327,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_indexing import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_indexops_spark.py
b/python/pyspark/pandas/tests/test_indexops_spark.py
index 275ef77f71b..8b0b5c87c9e 100644
--- a/python/pyspark/pandas/tests/test_indexops_spark.py
+++ b/python/pyspark/pandas/tests/test_indexops_spark.py
@@ -68,7 +68,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_indexops_spark import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_internal.py
b/python/pyspark/pandas/tests/test_internal.py
index 2ace222ed63..30a4bdcb66c 100644
--- a/python/pyspark/pandas/tests/test_internal.py
+++ b/python/pyspark/pandas/tests/test_internal.py
@@ -112,7 +112,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_internal import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_namespace.py
b/python/pyspark/pandas/tests/test_namespace.py
index 8f73c658467..c0bda11d98b 100644
--- a/python/pyspark/pandas/tests/test_namespace.py
+++ b/python/pyspark/pandas/tests/test_namespace.py
@@ -621,7 +621,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_namespace import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py
b/python/pyspark/pandas/tests/test_numpy_compat.py
index d16d9996ecd..fc6e3327827 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -188,7 +188,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_numpy_compat import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
index 71c393dcf34..734e2545d1f 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
@@ -2141,7 +2141,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_ops_on_diff_frames import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py
b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py
index 69621e49301..1bc1ab47723 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py
@@ -630,7 +630,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_ops_on_diff_frames_groupby import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git
a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
index 08f17745dfa..072a83d2945 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
@@ -99,7 +99,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_ops_on_diff_frames_groupby_expanding import
* # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git
a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
index 04ea448d800..e9a42e79abc 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
@@ -99,7 +99,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_ops_on_diff_frames_groupby_rolling import *
# noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_repr.py
b/python/pyspark/pandas/tests/test_repr.py
index 271ed0a14cb..d1ba46e63f8 100644
--- a/python/pyspark/pandas/tests/test_repr.py
+++ b/python/pyspark/pandas/tests/test_repr.py
@@ -178,7 +178,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_repr import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_resample.py
b/python/pyspark/pandas/tests/test_resample.py
index 56106940f1f..3b494e05e76 100644
--- a/python/pyspark/pandas/tests/test_resample.py
+++ b/python/pyspark/pandas/tests/test_resample.py
@@ -295,7 +295,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_resample import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_reshape.py
b/python/pyspark/pandas/tests/test_reshape.py
index 30550a9fba4..a7574a53885 100644
--- a/python/pyspark/pandas/tests/test_reshape.py
+++ b/python/pyspark/pandas/tests/test_reshape.py
@@ -483,7 +483,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_reshape import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_rolling.py
b/python/pyspark/pandas/tests/test_rolling.py
index be21bf16d40..6c31073d3f9 100644
--- a/python/pyspark/pandas/tests/test_rolling.py
+++ b/python/pyspark/pandas/tests/test_rolling.py
@@ -242,7 +242,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_rolling import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_scalars.py
b/python/pyspark/pandas/tests/test_scalars.py
index 0c8aa8508ff..00900dbdd91 100644
--- a/python/pyspark/pandas/tests/test_scalars.py
+++ b/python/pyspark/pandas/tests/test_scalars.py
@@ -47,7 +47,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_scalars import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_series.py
b/python/pyspark/pandas/tests/test_series.py
index e47f716ecfc..46a687b36c9 100644
--- a/python/pyspark/pandas/tests/test_series.py
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -3392,7 +3392,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_series import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_series_conversion.py
b/python/pyspark/pandas/tests/test_series_conversion.py
index bc83fdacbe2..79c2f1ff306 100644
--- a/python/pyspark/pandas/tests/test_series_conversion.py
+++ b/python/pyspark/pandas/tests/test_series_conversion.py
@@ -68,7 +68,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_series_conversion import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_series_datetime.py
b/python/pyspark/pandas/tests/test_series_datetime.py
index 1fe078e9726..1c392644edc 100644
--- a/python/pyspark/pandas/tests/test_series_datetime.py
+++ b/python/pyspark/pandas/tests/test_series_datetime.py
@@ -287,7 +287,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_series_datetime import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_series_string.py
b/python/pyspark/pandas/tests/test_series_string.py
index 0b778583e73..f82f57981f5 100644
--- a/python/pyspark/pandas/tests/test_series_string.py
+++ b/python/pyspark/pandas/tests/test_series_string.py
@@ -336,7 +336,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_series_string import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_spark_functions.py
b/python/pyspark/pandas/tests/test_spark_functions.py
index c18dc30240c..4da20f754d2 100644
--- a/python/pyspark/pandas/tests/test_spark_functions.py
+++ b/python/pyspark/pandas/tests/test_spark_functions.py
@@ -34,7 +34,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_spark_functions import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_sql.py
b/python/pyspark/pandas/tests/test_sql.py
index 5a5d6d484b9..4d4afb88828 100644
--- a/python/pyspark/pandas/tests/test_sql.py
+++ b/python/pyspark/pandas/tests/test_sql.py
@@ -100,7 +100,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_sql import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_stats.py
b/python/pyspark/pandas/tests/test_stats.py
index 4fb08ee69ed..fa7cff8f3cf 100644
--- a/python/pyspark/pandas/tests/test_stats.py
+++ b/python/pyspark/pandas/tests/test_stats.py
@@ -554,7 +554,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_stats import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_typedef.py
b/python/pyspark/pandas/tests/test_typedef.py
index 1bc5c8cfdd0..a5f2b2dc2b4 100644
--- a/python/pyspark/pandas/tests/test_typedef.py
+++ b/python/pyspark/pandas/tests/test_typedef.py
@@ -133,7 +133,7 @@ class TypeHintTests(unittest.TestCase):
pdf = pd.DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]})
- def func() -> pd.DataFrame[pdf.dtypes]: # type: ignore[name-defined]
+ def func() -> pd.DataFrame[pdf.dtypes]:
pass
expected = StructType([StructField("c0", LongType()),
StructField("c1", LongType())])
@@ -143,14 +143,14 @@ class TypeHintTests(unittest.TestCase):
pdf = pd.DataFrame({"a": [1, 2, 3], "b": pd.Categorical(["a", "b",
"c"])})
- def func() -> pd.Series[pdf.b.dtype]: # type: ignore[name-defined]
+ def func() -> pd.Series[pdf.b.dtype]:
pass
inferred = infer_return_type(func)
self.assertEqual(inferred.dtype, CategoricalDtype(categories=["a",
"b", "c"]))
self.assertEqual(inferred.spark_type, LongType())
- def func() -> pd.DataFrame[pdf.dtypes]: # type: ignore[name-defined]
+ def func() -> pd.DataFrame[pdf.dtypes]:
pass
expected = StructType([StructField("c0", LongType()),
StructField("c1", LongType())])
@@ -246,7 +246,7 @@ class TypeHintTests(unittest.TestCase):
pdf = pd.DataFrame({"a": ["a", 2, None]})
def try_infer_return_type():
- def f() -> pd.DataFrame[pdf.dtypes]: # type: ignore[name-defined]
+ def f() -> pd.DataFrame[pdf.dtypes]:
pass
infer_return_type(f)
@@ -254,7 +254,7 @@ class TypeHintTests(unittest.TestCase):
self.assertRaisesRegex(TypeError, "object.*not understood",
try_infer_return_type)
def try_infer_return_type():
- def f() -> pd.Series[pdf.a.dtype]: # type: ignore[name-defined]
+ def f() -> pd.Series[pdf.a.dtype]:
pass
infer_return_type(f)
@@ -293,7 +293,7 @@ class TypeHintTests(unittest.TestCase):
pdf = pd.DataFrame({"a": ["a", 2, None]})
def try_infer_return_type():
- def f() -> ps.DataFrame[pdf.dtypes]: # type: ignore[name-defined]
+ def f() -> ps.DataFrame[pdf.dtypes]:
pass
infer_return_type(f)
@@ -301,7 +301,7 @@ class TypeHintTests(unittest.TestCase):
self.assertRaisesRegex(TypeError, "object.*not understood",
try_infer_return_type)
def try_infer_return_type():
- def f() -> ps.Series[pdf.a.dtype]: # type: ignore[name-defined]
+ def f() -> ps.Series[pdf.a.dtype]:
pass
infer_return_type(f)
@@ -439,7 +439,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_typedef import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_utils.py
b/python/pyspark/pandas/tests/test_utils.py
index 11f560c6f55..cfbcb5ba0ad 100644
--- a/python/pyspark/pandas/tests/test_utils.py
+++ b/python/pyspark/pandas/tests/test_utils.py
@@ -121,7 +121,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_utils import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/pandas/tests/test_window.py
b/python/pyspark/pandas/tests/test_window.py
index 49779566c90..d8bc2775fa5 100644
--- a/python/pyspark/pandas/tests/test_window.py
+++ b/python/pyspark/pandas/tests/test_window.py
@@ -453,7 +453,7 @@ if __name__ == "__main__":
from pyspark.pandas.tests.test_window import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/resource/tests/test_resources.py
b/python/pyspark/resource/tests/test_resources.py
index b6babf3c6cb..81a4ea4f1d8 100644
--- a/python/pyspark/resource/tests/test_resources.py
+++ b/python/pyspark/resource/tests/test_resources.py
@@ -75,7 +75,7 @@ if __name__ == "__main__":
from pyspark.resource.tests.test_resources import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/connect/__init__.py
b/python/pyspark/sql/connect/__init__.py
index 3df96963f92..4a98368c817 100644
--- a/python/pyspark/sql/connect/__init__.py
+++ b/python/pyspark/sql/connect/__init__.py
@@ -18,5 +18,13 @@
"""Currently Spark Connect is very experimental and the APIs to interact with
Spark through this API are can be changed at any time without warning."""
-
from pyspark.sql.connect.dataframe import DataFrame # noqa: F401
+from pyspark.sql.pandas.utils import (
+ require_minimum_pandas_version,
+ require_minimum_pyarrow_version,
+ require_minimum_grpc_version,
+)
+
+require_minimum_pandas_version()
+require_minimum_pyarrow_version()
+require_minimum_grpc_version()
diff --git a/python/pyspark/sql/connect/client.py
b/python/pyspark/sql/connect/client.py
index 790f9980d5d..c4c74f5d6cc 100644
--- a/python/pyspark/sql/connect/client.py
+++ b/python/pyspark/sql/connect/client.py
@@ -20,7 +20,7 @@ import urllib.parse
import uuid
from typing import Iterable, Optional, Any, Union, List, Tuple, Dict
-import grpc # type: ignore
+import grpc
import pandas
import pyarrow as pa
diff --git a/python/pyspark/sql/pandas/utils.py
b/python/pyspark/sql/pandas/utils.py
index c51a90ca57a..ab62b955b63 100644
--- a/python/pyspark/sql/pandas/utils.py
+++ b/python/pyspark/sql/pandas/utils.py
@@ -73,6 +73,25 @@ def require_minimum_pyarrow_version() -> None:
)
+def require_minimum_grpc_version() -> None:
+ """Raise ImportError if minimum version of grpc is not installed"""
+ minimum_pandas_version = "1.48.1"
+
+ from distutils.version import LooseVersion
+
+ try:
+ import grpc
+ except ImportError as error:
+ raise ImportError(
+ "grpc >= %s must be installed; however, " "it was not found." %
minimum_pandas_version
+ ) from error
+ if LooseVersion(grpc.__version__) < LooseVersion(minimum_pandas_version):
+ raise ImportError(
+ "Pandas >= %s must be installed; however, "
+ "your version was %s." % (minimum_pandas_version, grpc.__version__)
+ )
+
+
def pyarrow_version_less_than_minimum(minimum_pyarrow_version: str) -> bool:
"""Return False if the installed pyarrow version is less than
minimum_pyarrow_version
or if pyarrow is not installed."""
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py
b/python/pyspark/sql/tests/connect/test_connect_basic.py
index c2a17c7786e..9d49cfd321c 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -14,36 +14,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any
import unittest
import shutil
-import numpy as np
import tempfile
-import grpc # type: ignore
-
-from pyspark.testing.sqlutils import have_pandas, SQLTestUtils
-
-if have_pandas:
- import pandas
-
+from pyspark.testing.sqlutils import SQLTestUtils
from pyspark.sql import SparkSession, Row
from pyspark.sql.types import StructType, StructField, LongType, StringType
+import pyspark.sql.functions
+from pyspark.testing.utils import ReusedPySparkTestCase
+from pyspark.testing.connectutils import should_test_connect,
connect_requirement_message
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
-if have_pandas:
+if should_test_connect:
+ import grpc
+ import pandas as pd
+ import numpy as np
from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
from pyspark.sql.connect.client import ChannelBuilder
from pyspark.sql.connect.column import Column
from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
from pyspark.sql.connect.function_builder import udf
from pyspark.sql.connect.functions import lit, col
- from pyspark.testing.pandasutils import PandasOnSparkTestCase
-else:
- from pyspark.testing.sqlutils import ReusedSQLTestCase as
PandasOnSparkTestCase # type: ignore
-from pyspark.sql.dataframe import DataFrame
-import pyspark.sql.functions
-from pyspark.testing.connectutils import should_test_connect,
connect_requirement_message
-from pyspark.testing.utils import ReusedPySparkTestCase
@unittest.skipIf(not should_test_connect, connect_requirement_message)
@@ -51,15 +43,8 @@ class SparkConnectSQLTestCase(PandasOnSparkTestCase,
ReusedPySparkTestCase, SQLT
"""Parent test fixture class for all Spark Connect related
test cases."""
- if have_pandas:
- connect: RemoteSparkSession
- tbl_name: str
- tbl_name_empty: str
- df_text: "DataFrame"
- spark: SparkSession
-
@classmethod
- def setUpClass(cls: Any):
+ def setUpClass(cls):
ReusedPySparkTestCase.setUpClass()
cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
cls.hive_available = True
@@ -80,12 +65,12 @@ class SparkConnectSQLTestCase(PandasOnSparkTestCase,
ReusedPySparkTestCase, SQLT
cls.spark_connect_load_test_data()
@classmethod
- def tearDownClass(cls: Any) -> None:
+ def tearDownClass(cls):
cls.spark_connect_clean_up_test_data()
ReusedPySparkTestCase.tearDownClass()
@classmethod
- def spark_connect_load_test_data(cls: Any):
+ def spark_connect_load_test_data(cls):
# Setup Remote Spark Session
cls.connect = RemoteSparkSession.builder.remote().getOrCreate()
df = cls.spark.createDataFrame([(x, f"{x}") for x in range(100)],
["id", "name"])
@@ -106,7 +91,7 @@ class SparkConnectSQLTestCase(PandasOnSparkTestCase,
ReusedPySparkTestCase, SQLT
empty_df.write.saveAsTable(cls.tbl_name_empty)
@classmethod
- def spark_connect_clean_up_test_data(cls: Any) -> None:
+ def spark_connect_clean_up_test_data(cls):
cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name))
cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name2))
cls.spark.sql("DROP TABLE IF EXISTS {}".format(cls.tbl_name_empty))
@@ -210,7 +195,7 @@ class SparkConnectTests(SparkConnectSQLTestCase):
def test_with_local_data(self):
"""SPARK-41114: Test creating a dataframe using local data"""
- pdf = pandas.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+ pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
df = self.connect.createDataFrame(pdf)
rows = df.filter(df.a == lit(3)).collect()
self.assertTrue(len(rows) == 1)
@@ -218,7 +203,7 @@ class SparkConnectTests(SparkConnectSQLTestCase):
self.assertEqual(rows[0][1], "c")
# Check correct behavior for empty DataFrame
- pdf = pandas.DataFrame({"a": []})
+ pdf = pd.DataFrame({"a": []})
with self.assertRaises(ValueError):
self.connect.createDataFrame(pdf)
@@ -1066,7 +1051,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.connect.test_connect_basic import * # noqa: F401
try:
- import xmlrunner # type: ignore
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/connect/test_connect_column.py
b/python/pyspark/sql/tests/connect/test_connect_column.py
index 734b0bbf226..c8979de6533 100644
--- a/python/pyspark/sql/tests/connect/test_connect_column.py
+++ b/python/pyspark/sql/tests/connect/test_connect_column.py
@@ -14,10 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
from pyspark.sql.tests.connect.test_connect_basic import
SparkConnectSQLTestCase
from pyspark.sql.types import StringType
-from pyspark.testing.sqlutils import have_pandas
from pyspark.sql.types import (
ByteType,
ShortType,
@@ -31,10 +29,11 @@ from pyspark.sql.types import (
BinaryType,
BooleanType,
)
+from pyspark.testing.connectutils import should_test_connect
-if have_pandas:
+if should_test_connect:
+ import pandas as pd
from pyspark.sql.connect.functions import lit
- import pandas
class SparkConnectTests(SparkConnectSQLTestCase):
@@ -88,11 +87,11 @@ class SparkConnectTests(SparkConnectSQLTestCase):
def test_simple_binary_expressions(self):
"""Test complex expression"""
df = self.connect.read.table(self.tbl_name)
- pd = df.select(df.id).where(df.id % lit(30) ==
lit(0)).sort(df.id.asc()).toPandas()
- self.assertEqual(len(pd.index), 4)
+ pdf = df.select(df.id).where(df.id % lit(30) ==
lit(0)).sort(df.id.asc()).toPandas()
+ self.assertEqual(len(pdf.index), 4)
- res = pandas.DataFrame(data={"id": [0, 30, 60, 90]})
- self.assert_(pd.equals(res), f"{pd.to_string()} != {res.to_string()}")
+ res = pd.DataFrame(data={"id": [0, 30, 60, 90]})
+ self.assert_(pdf.equals(res), f"{pdf.to_string()} !=
{res.to_string()}")
def test_cast(self):
df = self.connect.read.table(self.tbl_name)
@@ -150,7 +149,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.connect.test_connect_column import * # noqa: F401
try:
- import xmlrunner # type: ignore
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git
a/python/pyspark/sql/tests/connect/test_connect_column_expressions.py
b/python/pyspark/sql/tests/connect/test_connect_column_expressions.py
index 09e47657eb9..4e5d38bb078 100644
--- a/python/pyspark/sql/tests/connect/test_connect_column_expressions.py
+++ b/python/pyspark/sql/tests/connect/test_connect_column_expressions.py
@@ -15,22 +15,24 @@
# limitations under the License.
#
import uuid
-from typing import cast
import unittest
import decimal
import datetime
-from pyspark.testing.connectutils import PlanOnlyTestFixture
-from pyspark.testing.sqlutils import have_pandas, pandas_requirement_message
+from pyspark.testing.connectutils import (
+ PlanOnlyTestFixture,
+ should_test_connect,
+ connect_requirement_message,
+)
-if have_pandas:
+if should_test_connect:
from pyspark.sql.connect.proto import Expression as ProtoExpression
import pyspark.sql.connect.plan as p
from pyspark.sql.connect.column import Column
import pyspark.sql.connect.functions as fun
-@unittest.skipIf(not have_pandas, cast(str, pandas_requirement_message))
+@unittest.skipIf(not should_test_connect, connect_requirement_message)
class SparkConnectColumnExpressionSuite(PlanOnlyTestFixture):
def test_simple_column_expressions(self):
df = self.connect.with_plan(p.Read("table"))
@@ -228,7 +230,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.connect.test_connect_column_expressions import * #
noqa: F401
try:
- import xmlrunner # type: ignore
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py
b/python/pyspark/sql/tests/connect/test_connect_function.py
index d41f353d24f..ac257b4febe 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -14,22 +14,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any
import unittest
import tempfile
-from pyspark.testing.sqlutils import have_pandas, SQLTestUtils
-
from pyspark.sql import SparkSession
-
-if have_pandas:
- from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
- from pyspark.testing.pandasutils import PandasOnSparkTestCase
-else:
- from pyspark.testing.sqlutils import ReusedSQLTestCase as
PandasOnSparkTestCase # type: ignore
-from pyspark.sql.dataframe import DataFrame
+from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.testing.connectutils import should_test_connect,
connect_requirement_message
from pyspark.testing.utils import ReusedPySparkTestCase
+from pyspark.testing.sqlutils import SQLTestUtils
+
+if should_test_connect:
+ from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
@unittest.skipIf(not should_test_connect, connect_requirement_message)
@@ -37,15 +32,8 @@ class SparkConnectFuncTestCase(PandasOnSparkTestCase,
ReusedPySparkTestCase, SQL
"""Parent test fixture class for all Spark Connect related
test cases."""
- if have_pandas:
- connect: RemoteSparkSession
- tbl_name: str
- tbl_name_empty: str
- df_text: "DataFrame"
- spark: SparkSession
-
@classmethod
- def setUpClass(cls: Any):
+ def setUpClass(cls):
ReusedPySparkTestCase.setUpClass()
cls.tempdir = tempfile.NamedTemporaryFile(delete=False)
cls.hive_available = True
@@ -55,7 +43,7 @@ class SparkConnectFuncTestCase(PandasOnSparkTestCase,
ReusedPySparkTestCase, SQL
cls.connect = RemoteSparkSession.builder.remote().getOrCreate()
@classmethod
- def tearDownClass(cls: Any) -> None:
+ def tearDownClass(cls):
ReusedPySparkTestCase.tearDownClass()
@@ -63,7 +51,7 @@ class SparkConnectFunctionTests(SparkConnectFuncTestCase):
"""These test cases exercise the interface to the proto plan
generation but do not call Spark."""
- def compare_by_show(self, df1: Any, df2: Any):
+ def compare_by_show(self, df1, df2):
from pyspark.sql.dataframe import DataFrame as SDF
from pyspark.sql.connect.dataframe import DataFrame as CDF
diff --git a/python/pyspark/sql/tests/connect/test_connect_plan_only.py
b/python/pyspark/sql/tests/connect/test_connect_plan_only.py
index 7cd97f02d32..83e21e42bad 100644
--- a/python/pyspark/sql/tests/connect/test_connect_plan_only.py
+++ b/python/pyspark/sql/tests/connect/test_connect_plan_only.py
@@ -14,13 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import cast
import unittest
-from pyspark.testing.connectutils import PlanOnlyTestFixture
-from pyspark.testing.sqlutils import have_pandas, pandas_requirement_message
+from pyspark.testing.connectutils import (
+ PlanOnlyTestFixture,
+ should_test_connect,
+ connect_requirement_message,
+)
-if have_pandas:
+if should_test_connect:
import pyspark.sql.connect.proto as proto
from pyspark.sql.connect.column import Column
from pyspark.sql.connect.plan import WriteOperation
@@ -29,7 +31,7 @@ if have_pandas:
from pyspark.sql.types import StringType
-@unittest.skipIf(not have_pandas, cast(str, pandas_requirement_message))
+@unittest.skipIf(not should_test_connect, connect_requirement_message)
class SparkConnectTestsPlanOnly(PlanOnlyTestFixture):
"""These test cases exercise the interface to the proto plan
generation but do not call Spark."""
@@ -437,7 +439,7 @@ class SparkConnectTestsPlanOnly(PlanOnlyTestFixture):
self.assertIsNotNone(u)
expr = u("ThisCol", "ThatCol", "OtherCol")
self.assertTrue(isinstance(expr, Column))
- self.assertTrue(isinstance(cast(Column, expr)._expr,
UserDefinedFunction))
+ self.assertTrue(isinstance(expr._expr, UserDefinedFunction))
u_plan = expr.to_plan(self.connect)
self.assertIsNotNone(u_plan)
diff --git a/python/pyspark/sql/tests/connect/test_connect_select_ops.py
b/python/pyspark/sql/tests/connect/test_connect_select_ops.py
index 01d1819fdc6..7f8153f7fca 100644
--- a/python/pyspark/sql/tests/connect/test_connect_select_ops.py
+++ b/python/pyspark/sql/tests/connect/test_connect_select_ops.py
@@ -14,19 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import cast
import unittest
-from pyspark.testing.connectutils import PlanOnlyTestFixture
-from pyspark.testing.sqlutils import have_pandas, pandas_requirement_message
+from pyspark.testing.connectutils import (
+ PlanOnlyTestFixture,
+ should_test_connect,
+ connect_requirement_message,
+)
-if have_pandas:
+if should_test_connect:
from pyspark.sql.connect.functions import col
from pyspark.sql.connect.plan import Read
import pyspark.sql.connect.proto as proto
-@unittest.skipIf(not have_pandas, cast(str, pandas_requirement_message))
+@unittest.skipIf(not should_test_connect, connect_requirement_message)
class SparkConnectToProtoSuite(PlanOnlyTestFixture):
def test_select_with_columns_and_strings(self):
df = self.connect.with_plan(Read("table"))
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
b/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
index b3f4c7331d1..c3557f4eb5a 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
@@ -407,7 +407,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_cogrouped_map import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
index 7f27671cfe0..0044ae3c727 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
@@ -744,7 +744,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_grouped_map import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git
a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
index e75148e524c..655f0bf151d 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py
@@ -240,7 +240,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state import *
# noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_map.py
b/python/pyspark/sql/tests/pandas/test_pandas_map.py
index 7f996ca55a8..243cc36c678 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_map.py
@@ -207,7 +207,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_map import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf.py
index 077db2971ea..d6d861edb3e 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf.py
@@ -296,7 +296,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_udf import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
index aa844fc5fd5..155695f497b 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
@@ -551,7 +551,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_udf_grouped_agg import * #
noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
index 6580f839a86..a5b3bfc1643 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
@@ -1333,7 +1333,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_udf_scalar import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py
index 8c77ed4b775..3cdf83e2d06 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py
@@ -238,7 +238,7 @@ class PandasUDFTypeHintsTests(ReusedSQLTestCase):
df = self.spark.range(10).selectExpr("id", "id as v")
def plus_one(v: Union[pd.Series, pd.DataFrame]) -> pd.Series:
- return v + 1 # type: ignore[return-value]
+ return v + 1
plus_one = pandas_udf("long")(plus_one)
actual = df.select(plus_one(df.v).alias("plus_one"))
@@ -360,7 +360,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_udf_typehints import * # noqa:
#401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git
a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py
index a6d3bd608de..9b6751564c4 100644
---
a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py
+++
b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py
@@ -241,7 +241,7 @@ class
PandasUDFTypeHintsWithFutureAnnotationsTests(ReusedSQLTestCase):
df = self.spark.range(10).selectExpr("id", "id as v")
def plus_one(v: Union[pd.Series, pd.DataFrame]) -> pd.Series:
- return v + 1 # type: ignore[return-value]
+ return v + 1
plus_one = pandas_udf("long")(plus_one)
actual = df.select(plus_one(df.v).alias("plus_one"))
@@ -367,7 +367,7 @@ if __name__ == "__main__":
from
pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations
import * # noqa: #401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py
b/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py
index 07e10a58d2d..596742a23bf 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py
@@ -398,7 +398,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.pandas.test_pandas_udf_window import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/streaming/test_streaming.py
b/python/pyspark/sql/tests/streaming/test_streaming.py
index f170787ff7c..a67e493a7c0 100644
--- a/python/pyspark/sql/tests/streaming/test_streaming.py
+++ b/python/pyspark/sql/tests/streaming/test_streaming.py
@@ -653,7 +653,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.streaming.test_streaming import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/streaming/test_streaming_listener.py
b/python/pyspark/sql/tests/streaming/test_streaming_listener.py
index de345652540..c6667e25177 100644
--- a/python/pyspark/sql/tests/streaming/test_streaming_listener.py
+++ b/python/pyspark/sql/tests/streaming/test_streaming_listener.py
@@ -299,7 +299,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.streaming.test_streaming_listener import * # noqa:
F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_arrow_map.py
b/python/pyspark/sql/tests/test_arrow_map.py
index a4c948fea3c..6166cc5dcc8 100644
--- a/python/pyspark/sql/tests/test_arrow_map.py
+++ b/python/pyspark/sql/tests/test_arrow_map.py
@@ -35,7 +35,7 @@ if have_pandas:
@unittest.skipIf(
not have_pandas or not have_pyarrow,
- pandas_requirement_message or pyarrow_requirement_message, # type:
ignore[arg-type]
+ pandas_requirement_message or pyarrow_requirement_message,
)
class MapInArrowTests(ReusedSQLTestCase):
@classmethod
@@ -130,7 +130,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_arrow_map import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_catalog.py
b/python/pyspark/sql/tests/test_catalog.py
index 24cd67251a8..2eccfab72fd 100644
--- a/python/pyspark/sql/tests/test_catalog.py
+++ b/python/pyspark/sql/tests/test_catalog.py
@@ -398,7 +398,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_catalog import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_column.py
b/python/pyspark/sql/tests/test_column.py
index 2c4730fd812..236fb1b539d 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -192,7 +192,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_column import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_conf.py
b/python/pyspark/sql/tests/test_conf.py
index 4ea160818dd..a8fa59c0364 100644
--- a/python/pyspark/sql/tests/test_conf.py
+++ b/python/pyspark/sql/tests/test_conf.py
@@ -48,7 +48,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_conf import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_context.py
b/python/pyspark/sql/tests/test_context.py
index 508a829975e..b3818333148 100644
--- a/python/pyspark/sql/tests/test_context.py
+++ b/python/pyspark/sql/tests/test_context.py
@@ -193,7 +193,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_context import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_datasources.py
b/python/pyspark/sql/tests/test_datasources.py
index 30c18556229..80ab8a33162 100644
--- a/python/pyspark/sql/tests/test_datasources.py
+++ b/python/pyspark/sql/tests/test_datasources.py
@@ -198,7 +198,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_datasources import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_functions.py
b/python/pyspark/sql/tests/test_functions.py
index 55ef012b6d0..94cb3c4f1ee 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -1156,7 +1156,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_functions import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_group.py
b/python/pyspark/sql/tests/test_group.py
index 19f1a0148b7..19e1228d250 100644
--- a/python/pyspark/sql/tests/test_group.py
+++ b/python/pyspark/sql/tests/test_group.py
@@ -41,7 +41,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_group import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_pandas_sqlmetrics.py
b/python/pyspark/sql/tests/test_pandas_sqlmetrics.py
index d182bafd8b5..22a0e92e818 100644
--- a/python/pyspark/sql/tests/test_pandas_sqlmetrics.py
+++ b/python/pyspark/sql/tests/test_pandas_sqlmetrics.py
@@ -60,7 +60,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_pandas_sqlmetrics import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_readwriter.py
b/python/pyspark/sql/tests/test_readwriter.py
index 2e1bdb44248..4aa24fc2beb 100644
--- a/python/pyspark/sql/tests/test_readwriter.py
+++ b/python/pyspark/sql/tests/test_readwriter.py
@@ -217,7 +217,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_readwriter import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_serde.py
b/python/pyspark/sql/tests/test_serde.py
index 4e9d347da9e..e8017cfd38a 100644
--- a/python/pyspark/sql/tests/test_serde.py
+++ b/python/pyspark/sql/tests/test_serde.py
@@ -145,7 +145,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_serde import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_session.py
b/python/pyspark/sql/tests/test_session.py
index 80c05e1a3cb..dacaff4d2d2 100644
--- a/python/pyspark/sql/tests/test_session.py
+++ b/python/pyspark/sql/tests/test_session.py
@@ -404,7 +404,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_session import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_types.py
b/python/pyspark/sql/tests/test_types.py
index b1d2eccea43..bc7aafe5f04 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -1405,7 +1405,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_types import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/sql/tests/test_utils.py
b/python/pyspark/sql/tests/test_utils.py
index 954fe9f24af..080d88788b5 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -81,7 +81,7 @@ if __name__ == "__main__":
from pyspark.sql.tests.test_utils import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/streaming/tests/test_context.py
b/python/pyspark/streaming/tests/test_context.py
index 1e2c1531764..1afcc90b9e6 100644
--- a/python/pyspark/streaming/tests/test_context.py
+++ b/python/pyspark/streaming/tests/test_context.py
@@ -176,7 +176,7 @@ if __name__ == "__main__":
from pyspark.streaming.tests.test_context import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/streaming/tests/test_dstream.py
b/python/pyspark/streaming/tests/test_dstream.py
index a52d08a1b15..d37e64affb5 100644
--- a/python/pyspark/streaming/tests/test_dstream.py
+++ b/python/pyspark/streaming/tests/test_dstream.py
@@ -698,7 +698,7 @@ if __name__ == "__main__":
from pyspark.streaming.tests.test_dstream import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/streaming/tests/test_kinesis.py
b/python/pyspark/streaming/tests/test_kinesis.py
index 7b09f5b8f5d..7efd7a7d0c8 100644
--- a/python/pyspark/streaming/tests/test_kinesis.py
+++ b/python/pyspark/streaming/tests/test_kinesis.py
@@ -110,7 +110,7 @@ if __name__ == "__main__":
from pyspark.streaming.tests.test_kinesis import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/streaming/tests/test_listener.py
b/python/pyspark/streaming/tests/test_listener.py
index f881b2d201c..aeec278b38a 100644
--- a/python/pyspark/streaming/tests/test_listener.py
+++ b/python/pyspark/streaming/tests/test_listener.py
@@ -152,7 +152,7 @@ if __name__ == "__main__":
from pyspark.streaming.tests.test_listener import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/testing/connectutils.py
b/python/pyspark/testing/connectutils.py
index 1979b6eb723..efc118b572d 100644
--- a/python/pyspark/testing/connectutils.py
+++ b/python/pyspark/testing/connectutils.py
@@ -14,52 +14,66 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import typing
import os
-from typing import Any, Dict, Optional
import functools
import unittest
-from pyspark.testing.sqlutils import have_pandas
+from pyspark.testing.sqlutils import (
+ have_pandas,
+ have_pyarrow,
+ pandas_requirement_message,
+ pyarrow_requirement_message,
+)
-if have_pandas:
+
+grpc_requirement_message = None
+try:
+ import grpc
+except ImportError as e:
+ grpc_requirement_message = str(e)
+have_grpc = grpc_requirement_message is None
+
+connect_not_compiled_message = None
+if have_pandas and have_pyarrow and have_grpc:
from pyspark.sql.connect import DataFrame
from pyspark.sql.connect.plan import Read, Range, SQL
from pyspark.testing.utils import search_jar
- from pyspark.sql.connect.plan import LogicalPlan
from pyspark.sql.connect.session import SparkSession
connect_jar = search_jar("connector/connect/server",
"spark-connect-assembly-", "spark-connect")
+ existing_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
+ jars_args = "--jars %s" % connect_jar
+ plugin_args = "--conf
spark.plugins=org.apache.spark.sql.connect.SparkConnectPlugin"
+ os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join([jars_args, plugin_args,
existing_args])
else:
- connect_jar = None
-
-
-if connect_jar is None:
- connect_requirement_message = (
+ connect_not_compiled_message = (
"Skipping all Spark Connect Python tests as the optional Spark Connect
project was "
"not compiled into a JAR. To run these tests, you need to build Spark
with "
"'build/sbt package' or 'build/mvn package' before running this test."
)
-else:
- existing_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
- jars_args = "--jars %s" % connect_jar
- plugin_args = "--conf
spark.plugins=org.apache.spark.sql.connect.SparkConnectPlugin"
- os.environ["PYSPARK_SUBMIT_ARGS"] = " ".join([jars_args, plugin_args,
existing_args])
- connect_requirement_message = None # type: ignore
-should_test_connect = connect_requirement_message is None and have_pandas
+
+connect_requirement_message = (
+ pandas_requirement_message
+ or pyarrow_requirement_message
+ or grpc_requirement_message
+ or connect_not_compiled_message
+)
+should_test_connect: str = typing.cast(str, connect_requirement_message is
None)
class MockRemoteSession:
- def __init__(self) -> None:
- self.hooks: Dict[str, Any] = {}
+ def __init__(self):
+ self.hooks = {}
- def set_hook(self, name: str, hook: Any) -> None:
+ def set_hook(self, name, hook):
self.hooks[name] = hook
- def drop_hook(self, name: str) -> None:
+ def drop_hook(self, name):
self.hooks.pop(name)
- def __getattr__(self, item: str) -> Any:
+ def __getattr__(self, item):
if item not in self.hooks:
raise LookupError(f"{item} is not defined as a method hook in
MockRemoteSession")
return functools.partial(self.hooks[item])
@@ -67,43 +81,36 @@ class MockRemoteSession:
@unittest.skipIf(not should_test_connect, connect_requirement_message)
class PlanOnlyTestFixture(unittest.TestCase):
-
- connect: "MockRemoteSession"
- if have_pandas:
- session: SparkSession
-
@classmethod
- def _read_table(cls, table_name: str) -> "DataFrame":
- return DataFrame.withPlan(Read(table_name), cls.connect) # type:
ignore
+ def _read_table(cls, table_name):
+ return DataFrame.withPlan(Read(table_name), cls.connect)
@classmethod
- def _udf_mock(cls, *args, **kwargs) -> str:
+ def _udf_mock(cls, *args, **kwargs):
return "internal_name"
@classmethod
def _session_range(
cls,
- start: int,
- end: int,
- step: int = 1,
- num_partitions: Optional[int] = None,
- ) -> "DataFrame":
- return DataFrame.withPlan(
- Range(start, end, step, num_partitions), cls.connect # type:
ignore
- )
+ start,
+ end,
+ step=1,
+ num_partitions=None,
+ ):
+ return DataFrame.withPlan(Range(start, end, step, num_partitions),
cls.connect)
@classmethod
- def _session_sql(cls, query: str) -> "DataFrame":
- return DataFrame.withPlan(SQL(query), cls.connect) # type: ignore
+ def _session_sql(cls, query):
+ return DataFrame.withPlan(SQL(query), cls.connect)
if have_pandas:
@classmethod
- def _with_plan(cls, plan: LogicalPlan) -> "DataFrame":
- return DataFrame.withPlan(plan, cls.connect) # type: ignore
+ def _with_plan(cls, plan):
+ return DataFrame.withPlan(plan, cls.connect)
@classmethod
- def setUpClass(cls: Any) -> None:
+ def setUpClass(cls):
cls.connect = MockRemoteSession()
cls.session = SparkSession.builder.remote().getOrCreate()
cls.tbl_name = "test_connect_plan_only_table_1"
@@ -115,7 +122,7 @@ class PlanOnlyTestFixture(unittest.TestCase):
cls.connect.set_hook("with_plan", cls._with_plan)
@classmethod
- def tearDownClass(cls: Any) -> None:
+ def tearDownClass(cls):
cls.connect.drop_hook("register_udf")
cls.connect.drop_hook("readTable")
cls.connect.drop_hook("range")
diff --git a/python/pyspark/testing/pandasutils.py
b/python/pyspark/testing/pandasutils.py
index ad2f74e8af4..6a828f10026 100644
--- a/python/pyspark/testing/pandasutils.py
+++ b/python/pyspark/testing/pandasutils.py
@@ -22,11 +22,6 @@ import warnings
from contextlib import contextmanager
from distutils.version import LooseVersion
-import pandas as pd
-from pandas.api.types import is_list_like # type: ignore[attr-defined]
-from pandas.core.dtypes.common import is_numeric_dtype
-from pandas.testing import assert_frame_equal, assert_index_equal,
assert_series_equal
-
from pyspark import pandas as ps
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.indexes import Index
@@ -36,7 +31,7 @@ from pyspark.testing.sqlutils import ReusedSQLTestCase
tabulate_requirement_message = None
try:
- from tabulate import tabulate # noqa: F401
+ from tabulate import tabulate
except ImportError as e:
# If tabulate requirement is not satisfied, skip related tests.
tabulate_requirement_message = str(e)
@@ -44,7 +39,7 @@ have_tabulate = tabulate_requirement_message is None
matplotlib_requirement_message = None
try:
- import matplotlib # noqa: F401
+ import matplotlib
except ImportError as e:
# If matplotlib requirement is not satisfied, skip related tests.
matplotlib_requirement_message = str(e)
@@ -52,7 +47,7 @@ have_matplotlib = matplotlib_requirement_message is None
plotly_requirement_message = None
try:
- import plotly # noqa: F401
+ import plotly
except ImportError as e:
# If plotly requirement is not satisfied, skip related tests.
plotly_requirement_message = str(e)
@@ -72,6 +67,10 @@ class PandasOnSparkTestCase(ReusedSQLTestCase):
return lambda x: getattr(x, func)()
def assertPandasEqual(self, left, right, check_exact=True):
+ import pandas as pd
+ from pandas.core.dtypes.common import is_numeric_dtype
+ from pandas.testing import assert_frame_equal, assert_index_equal,
assert_series_equal
+
if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
try:
if LooseVersion(pd.__version__) >= LooseVersion("1.1"):
@@ -157,6 +156,8 @@ class PandasOnSparkTestCase(ReusedSQLTestCase):
- Compare floats rounding to the number of decimal places, 7 after
dropping missing values (NaN, NaT, None)
"""
+ import pandas as pd
+
if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
msg = (
"DataFrames are not almost equal: "
@@ -217,6 +218,9 @@ class PandasOnSparkTestCase(ReusedSQLTestCase):
:param almost: if this is enabled, the comparison is delegated to
`unittest`'s
`assertAlmostEqual`. See its documentation for more
details.
"""
+ import pandas as pd
+ from pandas.api.types import is_list_like
+
lobj = self._to_pandas(left)
robj = self._to_pandas(right)
if isinstance(lobj, (pd.DataFrame, pd.Series, pd.Index)):
diff --git a/python/pyspark/tests/test_appsubmit.py
b/python/pyspark/tests/test_appsubmit.py
index ff9f3e4f160..79b6b4fa91a 100644
--- a/python/pyspark/tests/test_appsubmit.py
+++ b/python/pyspark/tests/test_appsubmit.py
@@ -298,7 +298,7 @@ if __name__ == "__main__":
from pyspark.tests.test_appsubmit import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_broadcast.py
b/python/pyspark/tests/test_broadcast.py
index bc4587ffa64..90d3caa7369 100644
--- a/python/pyspark/tests/test_broadcast.py
+++ b/python/pyspark/tests/test_broadcast.py
@@ -188,7 +188,7 @@ if __name__ == "__main__":
from pyspark.tests.test_broadcast import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_conf.py
b/python/pyspark/tests/test_conf.py
index 6a7c7a05a99..cc9ff829092 100644
--- a/python/pyspark/tests/test_conf.py
+++ b/python/pyspark/tests/test_conf.py
@@ -36,7 +36,7 @@ if __name__ == "__main__":
from pyspark.tests.test_conf import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_context.py
b/python/pyspark/tests/test_context.py
index 1b63869562f..d819656f3b7 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -97,7 +97,7 @@ class AddFileTests(PySparkTestCase):
# this job fails due to `userlibrary` not being on the Python path:
# disable logging in log4j temporarily
def func(x):
- from userlibrary import UserClass # type: ignore
+ from userlibrary import UserClass
return UserClass().hello()
@@ -145,7 +145,7 @@ class AddFileTests(PySparkTestCase):
# To ensure that we're actually testing addPyFile's effects, check that
# this fails due to `userlibrary` not being on the Python path:
def func():
- from userlib import UserClass # type: ignore[import]
+ from userlib import UserClass
UserClass()
@@ -159,7 +159,7 @@ class AddFileTests(PySparkTestCase):
def test_overwrite_system_module(self):
self.sc.addPyFile(os.path.join(SPARK_HOME,
"python/test_support/SimpleHTTPServer.py"))
- import SimpleHTTPServer # type: ignore[import]
+ import SimpleHTTPServer
self.assertEqual("My Server", SimpleHTTPServer.__name__)
@@ -338,7 +338,7 @@ if __name__ == "__main__":
from pyspark.tests.test_context import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_daemon.py
b/python/pyspark/tests/test_daemon.py
index d4cb90c4e81..22196e5369b 100644
--- a/python/pyspark/tests/test_daemon.py
+++ b/python/pyspark/tests/test_daemon.py
@@ -81,7 +81,7 @@ if __name__ == "__main__":
from pyspark.tests.test_daemon import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_install_spark.py
b/python/pyspark/tests/test_install_spark.py
index cd1c424a85b..6f39a09ae18 100644
--- a/python/pyspark/tests/test_install_spark.py
+++ b/python/pyspark/tests/test_install_spark.py
@@ -142,7 +142,7 @@ if __name__ == "__main__":
from pyspark.tests.test_install_spark import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_join.py
b/python/pyspark/tests/test_join.py
index ce4c6e5dfe6..de1c260696e 100644
--- a/python/pyspark/tests/test_join.py
+++ b/python/pyspark/tests/test_join.py
@@ -61,7 +61,7 @@ if __name__ == "__main__":
from pyspark.tests.test_join import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_memory_profiler.py
b/python/pyspark/tests/test_memory_profiler.py
index cdb75e5b6aa..7bd7debe6ea 100644
--- a/python/pyspark/tests/test_memory_profiler.py
+++ b/python/pyspark/tests/test_memory_profiler.py
@@ -33,7 +33,7 @@ from pyspark.testing.utils import PySparkTestCase
@unittest.skipIf(not has_memory_profiler, "Must have memory-profiler
installed.")
-@unittest.skipIf(not have_pandas, pandas_requirement_message) # type: ignore
+@unittest.skipIf(not have_pandas, pandas_requirement_message)
class MemoryProfilerTests(PySparkTestCase):
def setUp(self):
self._old_sys_path = list(sys.path)
@@ -156,7 +156,7 @@ if __name__ == "__main__":
from pyspark.tests.test_memory_profiler import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_pin_thread.py
b/python/pyspark/tests/test_pin_thread.py
index 2874e09853f..dd291b8a0cc 100644
--- a/python/pyspark/tests/test_pin_thread.py
+++ b/python/pyspark/tests/test_pin_thread.py
@@ -171,7 +171,7 @@ if __name__ == "__main__":
from pyspark.tests.test_pin_thread import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_profiler.py
b/python/pyspark/tests/test_profiler.py
index 8a078d36b46..1db33b59b84 100644
--- a/python/pyspark/tests/test_profiler.py
+++ b/python/pyspark/tests/test_profiler.py
@@ -155,7 +155,7 @@ if __name__ == "__main__":
from pyspark.tests.test_profiler import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index 23e41d6c036..752b5d5599c 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -931,7 +931,7 @@ if __name__ == "__main__":
from pyspark.tests.test_rdd import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_rddbarrier.py
b/python/pyspark/tests/test_rddbarrier.py
index 18d618e3e14..dd3d2d6b364 100644
--- a/python/pyspark/tests/test_rddbarrier.py
+++ b/python/pyspark/tests/test_rddbarrier.py
@@ -44,7 +44,7 @@ if __name__ == "__main__":
from pyspark.tests.test_rddbarrier import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_rddsampler.py
b/python/pyspark/tests/test_rddsampler.py
index b504c4ab980..b98f2668cdc 100644
--- a/python/pyspark/tests/test_rddsampler.py
+++ b/python/pyspark/tests/test_rddsampler.py
@@ -58,7 +58,7 @@ if __name__ == "__main__":
from pyspark.tests.test_rddsampler import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_readwrite.py
b/python/pyspark/tests/test_readwrite.py
index d7086c4bce8..73f1025635c 100644
--- a/python/pyspark/tests/test_readwrite.py
+++ b/python/pyspark/tests/test_readwrite.py
@@ -360,7 +360,7 @@ if __name__ == "__main__":
from pyspark.tests.test_readwrite import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_serializers.py
b/python/pyspark/tests/test_serializers.py
index 0a89861a26f..230723e1057 100644
--- a/python/pyspark/tests/test_serializers.py
+++ b/python/pyspark/tests/test_serializers.py
@@ -108,7 +108,7 @@ class SerializationTestCase(unittest.TestCase):
def test_pickling_file_handles(self):
# to be corrected with SPARK-11160
try:
- import xmlrunner # type: ignore[import] # noqa: F401
+ import xmlrunner # noqa: F401
except ImportError:
ser = CloudPickleSerializer()
out1 = sys.stderr
diff --git a/python/pyspark/tests/test_shuffle.py
b/python/pyspark/tests/test_shuffle.py
index fb11a84f8af..4fb73607a2e 100644
--- a/python/pyspark/tests/test_shuffle.py
+++ b/python/pyspark/tests/test_shuffle.py
@@ -259,7 +259,7 @@ if __name__ == "__main__":
from pyspark.tests.test_shuffle import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_statcounter.py
b/python/pyspark/tests/test_statcounter.py
index b10fe7cd911..747f42e67b2 100644
--- a/python/pyspark/tests/test_statcounter.py
+++ b/python/pyspark/tests/test_statcounter.py
@@ -122,7 +122,7 @@ if __name__ == "__main__":
from pyspark.tests.test_statcounter import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_taskcontext.py
b/python/pyspark/tests/test_taskcontext.py
index b90a788ae2e..5d410aa57e7 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -342,7 +342,7 @@ if __name__ == "__main__":
from pyspark.tests.test_taskcontext import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_util.py
b/python/pyspark/tests/test_util.py
index 0ba9a5852e4..77f06721b10 100644
--- a/python/pyspark/tests/test_util.py
+++ b/python/pyspark/tests/test_util.py
@@ -89,7 +89,7 @@ if __name__ == "__main__":
from pyspark.tests.test_util import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
diff --git a/python/pyspark/tests/test_worker.py
b/python/pyspark/tests/test_worker.py
index 06ada8f81d5..703690bf7f9 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -263,7 +263,7 @@ if __name__ == "__main__":
from pyspark.tests.test_worker import * # noqa: F401
try:
- import xmlrunner # type: ignore[import]
+ import xmlrunner
testRunner = xmlrunner.XMLTestRunner(output="target/test-reports",
verbosity=2)
except ImportError:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
