This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 75b0eb2d6017 [SPARK-46213][PYTHON] Introduce `PySparkImportError` for
error framework
75b0eb2d6017 is described below
commit 75b0eb2d601763847507a5e715b3732db004544a
Author: Haejoon Lee <haejoon....@databricks.com>
AuthorDate: Sat Dec 2 13:15:35 2023 -0800
[SPARK-46213][PYTHON] Introduce `PySparkImportError` for error framework
### What changes were proposed in this pull request?
This PR proposes to introduce `PySparkImportError` for error framework.
### Why are the changes needed?
For better error handling.
### Does this PR introduce _any_ user-facing change?
No API changes, but it's improve the user-facing error messages.
### How was this patch tested?
The existing CI should pass
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44123 from itholic/pyspark_import_error.
Authored-by: Haejoon Lee <haejoon....@databricks.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
python/docs/source/reference/pyspark.errors.rst | 1 +
python/pyspark/errors/__init__.py | 2 ++
python/pyspark/errors/error_classes.py | 10 +++++++
python/pyspark/errors/exceptions/base.py | 6 ++++
python/pyspark/sql/connect/utils.py | 36 ++++++++++++++++-------
python/pyspark/sql/pandas/utils.py | 38 ++++++++++++++++++-------
6 files changed, 72 insertions(+), 21 deletions(-)
diff --git a/python/docs/source/reference/pyspark.errors.rst
b/python/docs/source/reference/pyspark.errors.rst
index d659657afc56..88cbd405b83d 100644
--- a/python/docs/source/reference/pyspark.errors.rst
+++ b/python/docs/source/reference/pyspark.errors.rst
@@ -43,6 +43,7 @@ Classes
PySparkRuntimeError
PySparkTypeError
PySparkValueError
+ PySparkImportError
PySparkIndexError
PythonException
QueryExecutionException
diff --git a/python/pyspark/errors/__init__.py
b/python/pyspark/errors/__init__.py
index d0a62537d347..923cb665d112 100644
--- a/python/pyspark/errors/__init__.py
+++ b/python/pyspark/errors/__init__.py
@@ -39,6 +39,7 @@ from pyspark.errors.exceptions.base import ( # noqa: F401
SparkNoSuchElementException,
PySparkTypeError,
PySparkValueError,
+ PySparkImportError,
PySparkIndexError,
PySparkAttributeError,
PySparkRuntimeError,
@@ -69,6 +70,7 @@ __all__ = [
"SparkNoSuchElementException",
"PySparkTypeError",
"PySparkValueError",
+ "PySparkImportError",
"PySparkIndexError",
"PySparkAttributeError",
"PySparkRuntimeError",
diff --git a/python/pyspark/errors/error_classes.py
b/python/pyspark/errors/error_classes.py
index 289b16c9b606..e1a93aa6be1a 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -667,6 +667,11 @@ ERROR_CLASSES_JSON = """
"Only a single trigger is allowed."
]
},
+ "PACKAGE_NOT_INSTALLED" : {
+ "message" : [
+ "<package_name> >= <minimum_version> must be installed; however, it was
not found."
+ ]
+ },
"PIPE_FUNCTION_EXITED" : {
"message" : [
"Pipe function `<func_name>` exited with error code <error_code>."
@@ -908,6 +913,11 @@ ERROR_CLASSES_JSON = """
"<operation> is not supported."
]
},
+ "UNSUPPORTED_PACKAGE_VERSION" : {
+ "message" : [
+ "<package_name> >= <minimum_version> must be installed; however, your
version is <current_version>."
+ ]
+ },
"UNSUPPORTED_PARAM_TYPE_FOR_HIGHER_ORDER_FUNCTION" : {
"message" : [
"Function `<func_name>` should use only POSITIONAL or POSITIONAL OR
KEYWORD arguments."
diff --git a/python/pyspark/errors/exceptions/base.py
b/python/pyspark/errors/exceptions/base.py
index c84ca17c3dbd..4a2b31418e29 100644
--- a/python/pyspark/errors/exceptions/base.py
+++ b/python/pyspark/errors/exceptions/base.py
@@ -258,3 +258,9 @@ class PySparkPicklingError(PySparkException, PicklingError):
"""
Wrapper class for pickle.PicklingError to support error classes.
"""
+
+
+class PySparkImportError(PySparkException, ImportError):
+ """
+ Wrapper class for ImportError to support error classes.
+ """
diff --git a/python/pyspark/sql/connect/utils.py
b/python/pyspark/sql/connect/utils.py
index fd85d75060b5..88f26202b0b2 100644
--- a/python/pyspark/sql/connect/utils.py
+++ b/python/pyspark/sql/connect/utils.py
@@ -18,6 +18,7 @@ import sys
from pyspark.loose_version import LooseVersion
from pyspark.sql.pandas.utils import require_minimum_pandas_version,
require_minimum_pyarrow_version
+from pyspark.errors import PySparkImportError
def check_dependencies(mod_name: str) -> None:
@@ -45,13 +46,21 @@ def require_minimum_grpc_version() -> None:
try:
import grpc
except ImportError as error:
- raise ImportError(
- f"grpcio >= {minimum_grpc_version} must be installed; however, it
was not found."
+ raise PySparkImportError(
+ error_class="PACKAGE_NOT_INSTALLED",
+ message_parameters={
+ "package_name:": "grpcio",
+ "minimum_version": str(minimum_grpc_version),
+ },
) from error
if LooseVersion(grpc.__version__) < LooseVersion(minimum_grpc_version):
- raise ImportError(
- f"grpcio >= {minimum_grpc_version} must be installed; however, "
- f"your version was {grpc.__version__}."
+ raise PySparkImportError(
+ error_class="UNSUPPORTED_PACKAGE_VERSION",
+ message_parameters={
+ "package_name:": "grpcio",
+ "minimum_version": str(minimum_grpc_version),
+ "current_version": str(grpc.__version__),
+ },
)
@@ -62,8 +71,12 @@ def require_minimum_grpcio_status_version() -> None:
try:
import grpc_status # noqa
except ImportError as error:
- raise ImportError(
- f"grpcio-status >= {minimum_grpc_version} must be installed;
however, it was not found."
+ raise PySparkImportError(
+ error_class="PACKAGE_NOT_INSTALLED",
+ message_parameters={
+ "package_name:": "grpcio-status",
+ "minimum_version": str(minimum_grpc_version),
+ },
) from error
@@ -74,9 +87,12 @@ def require_minimum_googleapis_common_protos_version() ->
None:
try:
import google.rpc # noqa
except ImportError as error:
- raise ImportError(
- f"googleapis-common-protos >= {minimum_common_protos_version} must
be installed; "
- "however, it was not found."
+ raise PySparkImportError(
+ error_class="PACKAGE_NOT_INSTALLED",
+ message_parameters={
+ "package_name:": "googleapis-common-protos",
+ "minimum_version": str(minimum_common_protos_version),
+ },
) from error
diff --git a/python/pyspark/sql/pandas/utils.py
b/python/pyspark/sql/pandas/utils.py
index b62be2081028..25152fd5cc1b 100644
--- a/python/pyspark/sql/pandas/utils.py
+++ b/python/pyspark/sql/pandas/utils.py
@@ -16,6 +16,7 @@
#
from pyspark.loose_version import LooseVersion
+from pyspark.errors import PySparkImportError
def require_minimum_pandas_version() -> None:
@@ -31,13 +32,21 @@ def require_minimum_pandas_version() -> None:
have_pandas = False
raised_error = error
if not have_pandas:
- raise ImportError(
- "Pandas >= %s must be installed; however, " "it was not found." %
minimum_pandas_version
+ raise PySparkImportError(
+ error_class="PACKAGE_NOT_INSTALLED",
+ message_parameters={
+ "package_name:": "Pandas",
+ "minimum_version": str(minimum_pandas_version),
+ },
) from raised_error
if LooseVersion(pandas.__version__) < LooseVersion(minimum_pandas_version):
- raise ImportError(
- "Pandas >= %s must be installed; however, "
- "your version was %s." % (minimum_pandas_version,
pandas.__version__)
+ raise PySparkImportError(
+ error_class="UNSUPPORTED_PACKAGE_VERSION",
+ message_parameters={
+ "package_name:": "Pandas",
+ "minimum_version": str(minimum_pandas_version),
+ "current_version": str(pandas.__version__),
+ },
)
@@ -56,14 +65,21 @@ def require_minimum_pyarrow_version() -> None:
have_arrow = False
raised_error = error
if not have_arrow:
- raise ImportError(
- "PyArrow >= %s must be installed; however, "
- "it was not found." % minimum_pyarrow_version
+ raise PySparkImportError(
+ error_class="PACKAGE_NOT_INSTALLED",
+ message_parameters={
+ "package_name:": "PyArrow",
+ "minimum_version": str(minimum_pyarrow_version),
+ },
) from raised_error
if LooseVersion(pyarrow.__version__) <
LooseVersion(minimum_pyarrow_version):
- raise ImportError(
- "PyArrow >= %s must be installed; however, "
- "your version was %s." % (minimum_pyarrow_version,
pyarrow.__version__)
+ raise PySparkImportError(
+ error_class="UNSUPPORTED_PACKAGE_VERSION",
+ message_parameters={
+ "package_name:": "PyArrow",
+ "minimum_version": str(minimum_pyarrow_version),
+ "current_version": str(pyarrow.__version__),
+ },
)
if os.environ.get("ARROW_PRE_0_15_IPC_FORMAT", "0") == "1":
raise RuntimeError(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
