This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-3-test by this push:
new 1ca02983ac6 [v3-3-test] Docs: clarify Databricks retry_delay uses
exponential backoff (#68622) (#70729)
1ca02983ac6 is described below
commit 1ca02983ac666602c3486a888c49ba43742879c5
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Jul 30 11:16:44 2026 +0200
[v3-3-test] Docs: clarify Databricks retry_delay uses exponential backoff
(#68622) (#70729)
(cherry picked from commit ee8bb6d26499af27340ba025c8d4113559f0071a)
Co-authored-by: sduvvuri26 <[email protected]>
---
.../databricks/docs/operators/repos_create.rst | 2 +-
.../databricks/docs/operators/repos_delete.rst | 2 +-
.../databricks/docs/operators/repos_update.rst | 2 +-
.../providers/databricks/hooks/databricks.py | 5 ++--
.../providers/databricks/hooks/databricks_base.py | 5 ++--
.../providers/databricks/operators/databricks.py | 33 ++++++++++++++--------
.../databricks/operators/databricks_repos.py | 18 ++++++++----
.../providers/databricks/triggers/databricks.py | 8 ++++--
8 files changed, 49 insertions(+), 26 deletions(-)
diff --git a/providers/databricks/docs/operators/repos_create.rst
b/providers/databricks/docs/operators/repos_create.rst
index fbbcca940c3..adc76a4f018 100644
--- a/providers/databricks/docs/operators/repos_create.rst
+++ b/providers/databricks/docs/operators/repos_create.rst
@@ -53,7 +53,7 @@ To use this operator you need to provide at least ``git_url``
parameter.
* - databricks_retry_limit: integer
- amount of times retry if the Databricks backend is unreachable.
* - databricks_retry_delay: decimal
- - number of seconds to wait between retries.
+ - minimum wait in seconds between retryable attempts (exponential backoff
by default).
Examples
--------
diff --git a/providers/databricks/docs/operators/repos_delete.rst
b/providers/databricks/docs/operators/repos_delete.rst
index 09d8880c344..cd58435ac03 100644
--- a/providers/databricks/docs/operators/repos_delete.rst
+++ b/providers/databricks/docs/operators/repos_delete.rst
@@ -45,7 +45,7 @@ To use this operator you need to provide either ``repo_path``
or ``repo_id``.
* - databricks_retry_limit: integer
- amount of times retry if the Databricks backend is unreachable.
* - databricks_retry_delay: decimal
- - number of seconds to wait between retries.
+ - minimum wait in seconds between retryable attempts (exponential backoff
by default).
Examples
--------
diff --git a/providers/databricks/docs/operators/repos_update.rst
b/providers/databricks/docs/operators/repos_update.rst
index 5af97e9751c..43a0402547f 100644
--- a/providers/databricks/docs/operators/repos_update.rst
+++ b/providers/databricks/docs/operators/repos_update.rst
@@ -50,7 +50,7 @@ To use this operator you need to provide either ``branch`` or
``tag`` and either
* - databricks_retry_limit: integer
- amount of times retry if the Databricks backend is unreachable.
* - databricks_retry_delay: decimal
- - number of seconds to wait between retries.
+ - minimum wait in seconds between retryable attempts (exponential backoff
by default).
Examples
--------
diff --git
a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py
b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py
index 7bba7634552..fa8141c989b 100644
--- a/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py
+++ b/providers/databricks/src/airflow/providers/databricks/hooks/databricks.py
@@ -274,8 +274,9 @@ class DatabricksHook(BaseDatabricksHook):
will wait before timing-out.
:param retry_limit: The number of times to retry the connection in case of
service outages.
- :param retry_delay: The number of seconds to wait between retries (it
- might be a floating point number).
+ :param retry_delay: Minimum wait in seconds between retryable attempts
when using the
+ default retry strategy. The wait uses exponential backoff (doubling
after each
+ failure, capped at ``2 ** retry_limit`` seconds). May be a floating
point number.
:param retry_args: An optional dictionary with arguments passed to
``tenacity.Retrying`` class.
"""
diff --git
a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
index 7ab51fc7dfa..f254bc24cf9 100644
---
a/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
+++
b/providers/databricks/src/airflow/providers/databricks/hooks/databricks_base.py
@@ -96,8 +96,9 @@ class BaseDatabricksHook(BaseHook):
will wait before timing-out.
:param retry_limit: The number of times to retry the connection in case of
service outages.
- :param retry_delay: The number of seconds to wait between retries (it
- might be a floating point number).
+ :param retry_delay: Minimum wait in seconds between retryable attempts
when using the
+ default retry strategy. The wait uses exponential backoff (doubling
after each
+ failure, capped at ``2 ** retry_limit`` seconds). May be a floating
point number.
:param retry_args: An optional dictionary with arguments passed to
``tenacity.Retrying`` class.
:param caller: The name of the operator that is calling the hook.
"""
diff --git
a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
index 4bae6b28f0a..25cbae3321c 100644
---
a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
+++
b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
@@ -406,8 +406,10 @@ class DatabricksCreateJobsOperator(BaseOperator):
this run. By default the operator will poll every 30 seconds.
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
.. note::
@@ -642,8 +644,10 @@ class DatabricksSubmitRunOperator(BaseOperator):
this run. By default the operator will poll every 30 seconds.
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
:param do_xcom_push: Whether we should push run_id and run_page_url to
xcom.
:param git_source: Optional specification of a remote git repository from
which
@@ -1058,8 +1062,10 @@ class DatabricksRunNowOperator(BaseOperator):
this run. By default, the operator will poll every 30 seconds.
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
:param do_xcom_push: Whether we should push run_id and run_page_url to
xcom.
:param wait_for_termination: if we should wait for termination of the job
run. ``True`` by default.
@@ -1291,8 +1297,10 @@ class
DatabricksSQLStatementsOperator(DatabricksSQLStatementsMixin, BaseOperator
this statement. By default the operator will poll every 30 seconds.
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
:param do_xcom_push: Whether we should push statement_id to xcom.:
:param timeout: The timeout for the Airflow task executing the SQL
statement. By default a value of 3600 seconds is used.
@@ -1449,7 +1457,8 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
:param databricks_task_key: An optional task_key used to refer to the task
by Databricks API. By
default this will be set to the hash of ``dag_id + task_id``.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
- :param databricks_retry_delay: Number of seconds to wait between retries.
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy (exponential backoff). May be a
floating point number.
:param databricks_retry_limit: Amount of times to retry if the Databricks
backend is unreachable.
:param deferrable: Whether to run the operator in the deferrable mode.
:param existing_cluster_id: ID for existing cluster on which to run this
task.
@@ -1783,7 +1792,8 @@ class
DatabricksNotebookOperator(DatabricksTaskBaseOperator):
https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsCreate
:param databricks_conn_id: The name of the Airflow connection to use.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
- :param databricks_retry_delay: Number of seconds to wait between retries.
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy (exponential backoff). May be a
floating point number.
:param databricks_retry_limit: Amount of times to retry if the Databricks
backend is unreachable.
:param deferrable: Whether to run the operator in the deferrable mode.
:param existing_cluster_id: ID for existing cluster on which to run this
task.
@@ -1930,7 +1940,8 @@ class DatabricksTaskOperator(DatabricksTaskBaseOperator):
:param task_config: The configuration of the task to be run on Databricks.
:param databricks_conn_id: The name of the Airflow connection to use.
:param databricks_retry_args: An optional dictionary with arguments passed
to ``tenacity.Retrying`` class.
- :param databricks_retry_delay: Number of seconds to wait between retries.
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy (exponential backoff). May be a
floating point number.
:param databricks_retry_limit: Amount of times to retry if the Databricks
backend is unreachable.
:param deferrable: Whether to run the operator in the deferrable mode.
:param existing_cluster_id: ID for existing cluster on which to run this
task.
diff --git
a/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py
b/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py
index cb513ea8873..1abb742efbc 100644
---
a/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py
+++
b/providers/databricks/src/airflow/providers/databricks/operators/databricks_repos.py
@@ -51,8 +51,10 @@ class DatabricksReposCreateOperator(BaseOperator):
connection and create the key ``host`` and leave the ``host`` field
empty. (templated)
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
"""
# Used in airflow.models.BaseOperator
@@ -201,8 +203,10 @@ class DatabricksReposUpdateOperator(BaseOperator):
connection and create the key ``host`` and leave the ``host`` field
empty. (templated)
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
"""
# Used in airflow.models.BaseOperator
@@ -275,8 +279,10 @@ class DatabricksReposDeleteOperator(BaseOperator):
connection and create the key ``host`` and leave the ``host`` field
empty. (templated)
:param databricks_retry_limit: Amount of times retry if the Databricks
backend is
unreachable. Its value must be greater than or equal to 1.
- :param databricks_retry_delay: Number of seconds to wait between retries
(it
- might be a floating point number).
+ :param databricks_retry_delay: Minimum wait in seconds between retryable
attempts when
+ using the default retry strategy. The wait uses exponential backoff
(doubling after
+ each failure, capped at ``2 ** databricks_retry_limit`` seconds). May
be a floating
+ point number.
"""
# Used in airflow.models.BaseOperator
diff --git
a/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py
b/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py
index 2bb626b9114..57d489550f2 100644
---
a/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py
+++
b/providers/databricks/src/airflow/providers/databricks/triggers/databricks.py
@@ -36,7 +36,9 @@ class DatabricksExecutionTrigger(BaseTrigger):
:param polling_period_seconds: Controls the rate of the poll for the
result of this run.
By default, the trigger will poll every 30 seconds.
:param retry_limit: The number of times to retry the connection in case of
service outages.
- :param retry_delay: The number of seconds to wait between retries.
+ :param retry_delay: Minimum wait in seconds between retryable attempts
when using the
+ default retry strategy. The wait uses exponential backoff (doubling
after each
+ failure, capped at ``2 ** retry_limit`` seconds). May be a floating
point number.
:param retry_args: An optional dictionary with arguments passed to
``tenacity.Retrying`` class.
:param run_page_url: The run page url.
:param repair_run: Repair the databricks run in case of failure.
@@ -138,7 +140,9 @@ class DatabricksSQLStatementExecutionTrigger(BaseTrigger):
:param polling_period_seconds: Controls the rate of the poll for the
result of this run.
By default, the trigger will poll every 30 seconds.
:param retry_limit: The number of times to retry the connection in case of
service outages.
- :param retry_delay: The number of seconds to wait between retries.
+ :param retry_delay: Minimum wait in seconds between retryable attempts
when using the
+ default retry strategy. The wait uses exponential backoff (doubling
after each
+ failure, capped at ``2 ** retry_limit`` seconds). May be a floating
point number.
:param retry_args: An optional dictionary with arguments passed to
``tenacity.Retrying`` class.
:param caller: The name of the operator that is calling the hook.
"""
