This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 31a48381f51 [SPARK-46230][PYTHON] Migrate `RetriesExceeded` into
PySpark error
31a48381f51 is described below
commit 31a48381f5139a51045a10df344df3ce7ad1adb7
Author: Haejoon Lee <haejoon....@databricks.com>
AuthorDate: Wed Dec 6 11:00:43 2023 -0800
[SPARK-46230][PYTHON] Migrate `RetriesExceeded` into PySpark error
### What changes were proposed in this pull request?
This PR proposes to migrate `RetriesExceeded` into PySpark error.
### Why are the changes needed?
All errors defined from PySpark should be inherits `PySparkException` to
keep the consistency of error messages generated from PySpark.
### Does this PR introduce _any_ user-facing change?
No, it's internal refactoring for better error handling.
### How was this patch tested?
The existing CI should pass.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44147 from itholic/retires_exception.
Authored-by: Haejoon Lee <haejoon....@databricks.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
python/docs/source/reference/pyspark.errors.rst | 1 +
python/pyspark/errors/__init__.py | 2 ++
python/pyspark/errors/error_classes.py | 5 +++++
python/pyspark/errors/exceptions/base.py | 7 +++++++
python/pyspark/sql/connect/client/retries.py | 11 ++---------
python/pyspark/sql/tests/connect/client/test_client.py | 2 +-
python/pyspark/sql/tests/connect/test_connect_basic.py | 3 ++-
7 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/python/docs/source/reference/pyspark.errors.rst
b/python/docs/source/reference/pyspark.errors.rst
index a4997506b41..270a8a8c716 100644
--- a/python/docs/source/reference/pyspark.errors.rst
+++ b/python/docs/source/reference/pyspark.errors.rst
@@ -48,6 +48,7 @@ Classes
PySparkIndexError
PythonException
QueryExecutionException
+ RetriesExceeded
SessionNotSameException
SparkRuntimeException
SparkUpgradeException
diff --git a/python/pyspark/errors/__init__.py
b/python/pyspark/errors/__init__.py
index 07033d21643..a4f64e85f87 100644
--- a/python/pyspark/errors/__init__.py
+++ b/python/pyspark/errors/__init__.py
@@ -46,6 +46,7 @@ from pyspark.errors.exceptions.base import ( # noqa: F401
PySparkAssertionError,
PySparkNotImplementedError,
PySparkPicklingError,
+ RetriesExceeded,
PySparkKeyError,
)
@@ -78,5 +79,6 @@ __all__ = [
"PySparkAssertionError",
"PySparkNotImplementedError",
"PySparkPicklingError",
+ "RetriesExceeded",
"PySparkKeyError",
]
diff --git a/python/pyspark/errors/error_classes.py
b/python/pyspark/errors/error_classes.py
index c93ffa94149..965fd04a913 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -813,6 +813,11 @@ ERROR_CLASSES_JSON = """
"Columns do not match in their data type: <mismatch>."
]
},
+ "RETRIES_EXCEEDED" : {
+ "message" : [
+ "The maximum number of retries has been exceeded."
+ ]
+ },
"SCHEMA_MISMATCH_FOR_PANDAS_UDF" : {
"message" : [
"Result vector from pandas_udf was not the required length: expected
<expected>, got <actual>."
diff --git a/python/pyspark/errors/exceptions/base.py
b/python/pyspark/errors/exceptions/base.py
index b7d8ed88ec0..b60800da3ff 100644
--- a/python/pyspark/errors/exceptions/base.py
+++ b/python/pyspark/errors/exceptions/base.py
@@ -260,6 +260,13 @@ class PySparkPicklingError(PySparkException,
PicklingError):
"""
+class RetriesExceeded(PySparkException):
+ """
+ Represents an exception which is considered retriable, but retry limits
+ were exceeded
+ """
+
+
class PySparkKeyError(PySparkException, KeyError):
"""
Wrapper class for KeyError to support error classes.
diff --git a/python/pyspark/sql/connect/client/retries.py
b/python/pyspark/sql/connect/client/retries.py
index 88fc3fe1ffd..44e5e1834a2 100644
--- a/python/pyspark/sql/connect/client/retries.py
+++ b/python/pyspark/sql/connect/client/retries.py
@@ -22,7 +22,7 @@ import typing
from typing import Optional, Callable, Generator, List, Type
from types import TracebackType
from pyspark.sql.connect.client.logging import logger
-from pyspark.errors import PySparkRuntimeError
+from pyspark.errors import PySparkRuntimeError, RetriesExceeded
"""
This module contains retry system. The system is designed to be
@@ -233,7 +233,7 @@ class Retrying:
# Exceeded retries
logger.debug(f"Given up on retrying. error: {repr(exception)}")
- raise RetriesExceeded from exception
+ raise RetriesExceeded(error_class="RETRIES_EXCEEDED",
message_parameters={}) from exception
def __iter__(self) -> Generator[AttemptManager, None, None]:
"""
@@ -315,10 +315,3 @@ class DefaultPolicy(RetryPolicy):
return True
return False
-
-
-class RetriesExceeded(Exception):
- """
- Represents an exception which is considered retriable, but retry limits
- were exceeded
- """
diff --git a/python/pyspark/sql/tests/connect/client/test_client.py
b/python/pyspark/sql/tests/connect/client/test_client.py
index 12e690c3a30..1ff3a8ea3e8 100644
--- a/python/pyspark/sql/tests/connect/client/test_client.py
+++ b/python/pyspark/sql/tests/connect/client/test_client.py
@@ -31,9 +31,9 @@ if should_test_connect:
from pyspark.sql.connect.client.retries import (
Retrying,
DefaultPolicy,
- RetriesExceeded,
)
from pyspark.sql.connect.client.reattach import
ExecutePlanResponseReattachableIterator
+ from pyspark.errors import RetriesExceeded
import pyspark.sql.connect.proto as proto
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py
b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 5e0cf535391..2431b948f9d 100755
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -31,10 +31,11 @@ from pyspark.errors import (
PySparkTypeError,
PySparkException,
PySparkValueError,
+ RetriesExceeded,
)
from pyspark.errors.exceptions.base import SessionNotSameException
from pyspark.sql import SparkSession as PySparkSession, Row
-from pyspark.sql.connect.client.retries import RetryPolicy, RetriesExceeded
+from pyspark.sql.connect.client.retries import RetryPolicy
from pyspark.sql.types import (
StructType,
StructField,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
