This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f6225620e23 Include last error cause in Databricks API when retry is
exhausted (#69238)
f6225620e23 is described below
commit f6225620e238c817dc8008c46fdec9eecc164664
Author: Amogh Desai <[email protected]>
AuthorDate: Mon Jul 6 14:29:59 2026 +0530
Include last error cause in Databricks API when retry is exhausted (#69238)
---
.../providers/databricks/hooks/databricks_base.py | 37 +++++++++++++++++-----
.../unit/databricks/hooks/test_databricks_base.py | 5 ++-
2 files changed, 33 insertions(+), 9 deletions(-)
diff --git
a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
index d66122063b2..c90d10cf69a 100644
---
a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
+++
b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
@@ -92,6 +92,15 @@ class DatabricksProxyConfigurationError(AirflowException):
"""Raised when Databricks connection proxy configuration is invalid."""
+def _describe_last_retry_error(retry_error: RetryError) -> str:
+ """Best-effort description of a RetryError last exception, some exceptions
raise from __str__ when partially initialized."""
+ last_exc = retry_error.last_attempt.exception()
+ try:
+ return str(last_exc)
+ except Exception:
+ return repr(last_exc)
+
+
class BaseDatabricksHook(BaseHook):
"""
Base for interaction with Databricks.
@@ -325,8 +334,11 @@ class BaseDatabricksHook(BaseHook):
self._is_oauth_token_valid(jsn)
self.oauth_tokens[resource] = jsn
break
- except RetryError:
- raise AirflowException(f"API requests to Databricks failed
{self.retry_limit} times. Giving up.")
+ except RetryError as e:
+ raise AirflowException(
+ f"API requests to Databricks failed {self.retry_limit} times "
+ f"(last error: {_describe_last_retry_error(e)}). Giving up."
+ ) from e
except requests_exceptions.HTTPError as e:
msg = f"Response: {e.response.content.decode()}, Status Code:
{e.response.status_code}"
raise AirflowException(msg)
@@ -363,8 +375,11 @@ class BaseDatabricksHook(BaseHook):
self._is_oauth_token_valid(jsn)
self.oauth_tokens[resource] = jsn
break
- except RetryError:
- raise AirflowException(f"API requests to Databricks failed
{self.retry_limit} times. Giving up.")
+ except RetryError as e:
+ raise AirflowException(
+ f"API requests to Databricks failed {self.retry_limit} times "
+ f"(last error: {_describe_last_retry_error(e)}). Giving up."
+ ) from e
except requests_exceptions.HTTPError as e:
msg = f"Response: {e.response.content.decode()}, Status Code:
{e.response.status_code}"
raise AirflowException(msg)
@@ -1213,8 +1228,11 @@ class BaseDatabricksHook(BaseHook):
self.log.debug("Response text: %s", response.text)
response.raise_for_status()
return response.json()
- except RetryError:
- raise AirflowException(f"API requests to Databricks failed
{self.retry_limit} times. Giving up.")
+ except RetryError as e:
+ raise AirflowException(
+ f"API requests to Databricks failed {self.retry_limit} times "
+ f"(last error: {_describe_last_retry_error(e)}). Giving up."
+ ) from e
except requests_exceptions.HTTPError as e:
if wrap_http_errors:
msg = f"Response: {e.response.content.decode()}, Status Code:
{e.response.status_code}"
@@ -1277,8 +1295,11 @@ class BaseDatabricksHook(BaseHook):
self.log.debug("Response text: %s", response.text)
response.raise_for_status()
return await response.json()
- except RetryError:
- raise AirflowException(f"API requests to Databricks failed
{self.retry_limit} times. Giving up.")
+ except RetryError as e:
+ raise AirflowException(
+ f"API requests to Databricks failed {self.retry_limit} times "
+ f"(last error: {_describe_last_retry_error(e)}). Giving up."
+ ) from e
except aiohttp.ClientResponseError as err:
raise DatabricksApiError(
f"Response: {err.message}, Status Code: {err.status}",
http_status_code=err.status
diff --git
a/providers/databricks/tests/unit/databricks/hooks/test_databricks_base.py
b/providers/databricks/tests/unit/databricks/hooks/test_databricks_base.py
index 393107497d7..2c976312811 100644
--- a/providers/databricks/tests/unit/databricks/hooks/test_databricks_base.py
+++ b/providers/databricks/tests/unit/databricks/hooks/test_databricks_base.py
@@ -238,7 +238,10 @@ class TestBaseDatabricksHook:
hook.user_agent_header = {"User-Agent": "test-agent"}
resource = ""
- with pytest.raises(AirflowException, match="API requests to Databricks
failed 2 times. Giving up."):
+ with pytest.raises(
+ AirflowException,
+ match=r"API requests to Databricks failed 2 times \(last error:
Connection failed\)\. Giving up\.",
+ ):
hook._get_sp_token(resource)
@pytest.mark.asyncio
