This is an automated email from the ASF dual-hosted git repository. yikun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 12a77bb22f1 [SPARK-41107][PYTHON][INFRA][TESTS] Install memory-profiler in the CI 12a77bb22f1 is described below commit 12a77bb22f1689e361a5efe2d7000aead74ebc43 Author: Xinrong Meng <xinr...@apache.org> AuthorDate: Fri Nov 18 17:12:39 2022 +0800 [SPARK-41107][PYTHON][INFRA][TESTS] Install memory-profiler in the CI ### What changes were proposed in this pull request? Install [memory-profiler](https://pypi.org/project/memory-profiler/) in the CI in order to enable memory profiling tests. ### Why are the changes needed? That's a sub-task of [SPARK-40281](https://issues.apache.org/jira/browse/SPARK-40281) Memory Profiler on Executors. PySpark memory profiler depends on memory-profiler. The PR proposes to install memory-profiler in the CI to enable related tests. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. Closes #38611 from xinrong-meng/ci_mp. Lead-authored-by: Xinrong Meng <xinr...@apache.org> Co-authored-by: Yikun Jiang <yikunk...@gmail.com> Signed-off-by: Yikun Jiang <yikunk...@gmail.com> --- dev/infra/Dockerfile | 3 +++ python/pyspark/tests/test_memory_profiler.py | 8 +------- python/pyspark/tests/test_profiler.py | 2 ++ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 96b20894b87..a6331c2ead4 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -68,3 +68,6 @@ ENV R_LIBS_SITE "/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library # Add Python deps for Spark Connect. RUN python3.9 -m pip install grpcio protobuf + +# SPARK-41186: Move memory-profiler to pyspark deps install when mlfow doctest test fix +RUN python3.9 -m pip install 'memory-profiler==0.60.0' diff --git a/python/pyspark/tests/test_memory_profiler.py b/python/pyspark/tests/test_memory_profiler.py index 7da82dccb37..3dc8ce4ce22 100644 --- a/python/pyspark/tests/test_memory_profiler.py +++ b/python/pyspark/tests/test_memory_profiler.py @@ -27,17 +27,11 @@ from unittest import mock import pandas as pd from pyspark import SparkConf, SparkContext +from pyspark.profiler import has_memory_profiler from pyspark.sql import SparkSession from pyspark.sql.functions import pandas_udf, udf from pyspark.testing.utils import PySparkTestCase -try: - import memory_profiler # type: ignore[import] # noqa: F401 - - has_memory_profiler = True -except Exception: - has_memory_profiler = False - @unittest.skipIf(not has_memory_profiler, "Must have memory-profiler installed.") class MemoryProfilerTests(PySparkTestCase): diff --git a/python/pyspark/tests/test_profiler.py b/python/pyspark/tests/test_profiler.py index ceae904ca6f..8a078d36b46 100644 --- a/python/pyspark/tests/test_profiler.py +++ b/python/pyspark/tests/test_profiler.py @@ -22,6 +22,7 @@ import unittest from io import StringIO from pyspark import SparkConf, SparkContext, BasicProfiler +from pyspark.profiler import has_memory_profiler from pyspark.sql import SparkSession from pyspark.sql.functions import udf from pyspark.sql.utils import PythonException @@ -126,6 +127,7 @@ class ProfilerTests2(unittest.TestCase): finally: sc.stop() + @unittest.skipIf(has_memory_profiler, "Test when memory-profiler is not installed.") def test_no_memory_profile_installed(self): sc = SparkContext( conf=SparkConf() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org