This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8a016b710ff Support durable execution in Databricks task operators
(#76295)
8a016b710ff is described below
commit 8a016b710ff8d3e69451c871469e6c526850e223
Author: Amogh Desai <[email protected]>
AuthorDate: Mon Nov 16 10:24:58 2026 +0530
Support durable execution in Databricks task operators (#76295)
---
providers/databricks/docs/operators/notebook.rst | 55 ++-
providers/databricks/docs/operators/run_now.rst | 21 +-
providers/databricks/docs/operators/submit_run.rst | 21 +-
providers/databricks/docs/operators/task.rst | 55 ++-
.../providers/databricks/operators/databricks.py | 207 ++++++++----
.../unit/databricks/operators/test_databricks.py | 374 +++++++++++++++++++--
6 files changed, 617 insertions(+), 116 deletions(-)
diff --git a/providers/databricks/docs/operators/notebook.rst
b/providers/databricks/docs/operators/notebook.rst
index 3c8ec27b8d3..57d34269361 100644
--- a/providers/databricks/docs/operators/notebook.rst
+++ b/providers/databricks/docs/operators/notebook.rst
@@ -67,7 +67,9 @@ whole Airflow task:
)
Airflow ``retries`` behaves differently depending on where the operator runs.
For a standalone
-operator, each retry submits a new Databricks run. Inside a
+operator, a retry reconnects to the run the previous try submitted only when
that try was lost
+while the run was still going; a run that already reached a terminal failure
on Databricks is
+replaced by a fresh submit (see `Durable execution`_ below). Inside a
:class:`~airflow.providers.databricks.operators.databricks_workflow.DatabricksWorkflowTaskGroup`,
the Airflow task monitors a sub-run that was already submitted by the workflow
launch task, so a
retry only re-polls the terminal sub-run. Use ``max_retries`` to retry
Databricks work inside a
@@ -80,3 +82,54 @@ attempt is observed, so downstream failure handling is not
delayed by long-runni
Only unlimited retries (``max_retries=-1``) keep the Airflow task waiting (or
deferring) until the
parent workflow run reaches a terminal state, because Databricks may still
launch a retry attempt
under the same ``task_key`` until then. Sibling tasks in the run continue
independently.
+
+
+Durable execution
+-----------------
+
+Outside a
+:class:`~airflow.providers.databricks.operators.databricks_workflow.DatabricksWorkflowTaskGroup`,
+this operator submits a run and then waits for it, on the worker or in the
Triggerer when
+``deferrable=True``. By default it runs in a *durable* mode: the Databricks
run id is persisted to
+:doc:`task state store <apache-airflow:core-concepts/task-state-store>` before
the wait begins, so if
+the worker crashes or is preempted and the task is retried, the operator
reconnects to the run that
+is already executing on Databricks instead of submitting a duplicate.
+
+On retry the operator checks the prior run's state:
+
+* if it is still running, the operator reconnects and continues polling
+* if it already succeeded, the operator returns immediately without
resubmitting
+* if it failed terminally, the operator submits a fresh run
+
+Pass ``durable=False`` to always submit a fresh run on retry.
+
+Unlike
:class:`~airflow.providers.databricks.operators.databricks.DatabricksSubmitRunOperator`
and
+:class:`~airflow.providers.databricks.operators.databricks.DatabricksRunNowOperator`,
where
+``durable`` has no effect in deferrable mode, durable execution here also
covers
+``deferrable=True``: the run id is stored before the task defers, so a try
lost while the Triggerer
+was waiting still reconnects on the next one.
+
+Durable execution requires Airflow 3.3 or newer, since it relies on the task
state store. On
+earlier Airflow versions the flag is a no-op and the operator always submits a
fresh run on retry.
+If the task state store is unavailable at runtime, the operator logs that
crash recovery is
+disabled and behaves the same way.
+
+Inside a ``DatabricksWorkflowTaskGroup`` nothing changes: the run is submitted
by the workflow
+launch task and this operator only monitors its own task within that shared
run, so a retry already
+reconnects rather than resubmitting.
+
+``[state_store] default_retention_days`` (30 days by default) is stamped on
the run id when it is
+stored, so a retry that comes later than that finds nothing and submits a
fresh run instead of
+reconnecting, whatever schedule ``airflow state-store clean`` runs on. Size
that setting against
+your longest ``retry_delay``. See
+:doc:`apache-airflow:administration-and-deployment/task-and-asset-state-store-cleanup`.
+
+Clearing a task discards the stored run id by default, so the next attempt
submits a fresh run.
+Set ``keep_task_state`` when clearing to keep it instead.
+:doc:`apache-airflow:core-concepts/resumable-tasks` lists which clear paths
discard it and
+describes the ``[state_store] clear_on_success`` setting.
+
+Where the id is kept, clearing behaves like a retry rather than a fresh start.
If the stored run
+already succeeded, the next attempt reads it back, re-pushes its XComs and
returns without
+submitting anything. If the run is still going and
``wait_for_termination=False``, the attempt
+reconnects to it and returns immediately, also without submitting.
diff --git a/providers/databricks/docs/operators/run_now.rst
b/providers/databricks/docs/operators/run_now.rst
index 64345c8f135..82a308f436e 100644
--- a/providers/databricks/docs/operators/run_now.rst
+++ b/providers/databricks/docs/operators/run_now.rst
@@ -137,17 +137,16 @@ Airflow versions the flag is a no-op and the operator
always triggers a fresh ru
exactly as before. If the task state store is unavailable at runtime, the
operator logs that crash
recovery is disabled and behaves the same way.
-Like the persisted state itself, the stored run id isn't deleted
automatically, that only happens
-when someone runs ``airflow state-store clean``. If a task's ``retry_delay``
is longer than
-``[state_store] default_retention_days`` (30 days by default) and cleanup runs
in between, the
-run id won't be there for the next retry, and the operator will trigger a
fresh run instead of
-reconnecting. Avoid running cleanup on a schedule shorter than your longest
``retry_delay``.
-
-Clearing a task is treated the same as a retry, which matters specifically for
a task whose run
-already succeeded: clearing does not delete the stored run id, so the next
attempt reads it back
-and returns immediately without triggering a new run. See
-:doc:`apache-airflow:core-concepts/resumable-tasks` for why, and for the
-``[state_store] clear_on_success`` setting that restores "clearing always
resubmits."
+``[state_store] default_retention_days`` (30 days by default) is stamped on
the run id when it is
+stored, so a retry that comes later than that finds nothing and triggers a
fresh run instead of
+reconnecting, whatever schedule ``airflow state-store clean`` runs on. Size
that setting against
+your longest ``retry_delay``.
+
+Clearing a task discards the stored run id by default, so the next attempt
triggers a fresh run.
+Set ``keep_task_state`` when clearing to keep it instead, and the attempt is
then treated the same
+as a retry: for a task whose run already succeeded, it reads the run back and
returns immediately
+without triggering a new one.
:doc:`apache-airflow:core-concepts/resumable-tasks` lists which
+clear paths discard the id and describes the ``[state_store]
clear_on_success`` setting.
This is most reliable for deferred tasks (``deferrable=True``); clearing a
task that's actively
polling synchronously can cancel the run via ``on_kill`` before the next
attempt gets a chance to
diff --git a/providers/databricks/docs/operators/submit_run.rst
b/providers/databricks/docs/operators/submit_run.rst
index 6964e96e79e..db23b2074d8 100644
--- a/providers/databricks/docs/operators/submit_run.rst
+++ b/providers/databricks/docs/operators/submit_run.rst
@@ -205,17 +205,16 @@ earlier Airflow versions the flag is a no-op and the
operator always submits a f
run on retry, exactly as before. If the task state store is unavailable at
runtime, the
operator logs that crash recovery is disabled and behaves the same way.
-Like the persisted state itself, the stored run id isn't deleted
automatically, that only happens
-when someone runs ``airflow state-store clean``. If a task's ``retry_delay``
is longer than
-``[state_store] default_retention_days`` (30 days by default) and cleanup runs
in between, the
-run id won't be there for the next retry, and the operator will submit a fresh
run instead of
-reconnecting. Avoid running cleanup on a schedule shorter than your longest
``retry_delay``.
-
-Clearing a task is treated the same as a retry, which matters specifically for
a task whose run
-already succeeded: clearing does not delete the stored run id, so the next
attempt reads it back
-and returns immediately without submitting a new run. See
-:doc:`apache-airflow:core-concepts/resumable-tasks` for why, and for the
-``[state_store] clear_on_success`` setting that restores "clearing always
resubmits."
+``[state_store] default_retention_days`` (30 days by default) is stamped on
the run id when it is
+stored, so a retry that comes later than that finds nothing and submits a
fresh run instead of
+reconnecting, whatever schedule ``airflow state-store clean`` runs on. Size
that setting against
+your longest ``retry_delay``.
+
+Clearing a task discards the stored run id by default, so the next attempt
submits a fresh run.
+Set ``keep_task_state`` when clearing to keep it instead, and the attempt is
then treated the same
+as a retry: for a task whose run already succeeded, it reads the run back and
returns immediately
+without submitting a new one.
:doc:`apache-airflow:core-concepts/resumable-tasks` lists which
+clear paths discard the id and describes the ``[state_store]
clear_on_success`` setting.
This is most reliable for deferred tasks (``deferrable=True``); clearing a
task that's actively
polling synchronously can cancel the run via ``on_kill`` before the next
attempt gets a chance to
diff --git a/providers/databricks/docs/operators/task.rst
b/providers/databricks/docs/operators/task.rst
index b327aa20ccf..100c6a74fed 100644
--- a/providers/databricks/docs/operators/task.rst
+++ b/providers/databricks/docs/operators/task.rst
@@ -58,7 +58,9 @@ whole Airflow task. You can set the same fields directly in
``task_config``. Whe
operator parameter takes precedence. If a field is unset, Databricks uses its
default.
Airflow ``retries`` behaves differently depending on where the operator runs.
For a standalone
-operator, each retry submits a new Databricks run. Inside a
+operator, a retry reconnects to the run the previous try submitted only when
that try was lost
+while the run was still going; a run that already reached a terminal failure
on Databricks is
+replaced by a fresh submit (see `Durable execution`_ below). Inside a
:class:`~airflow.providers.databricks.operators.databricks_workflow.DatabricksWorkflowTaskGroup`,
the Airflow task monitors a sub-run that was already submitted by the workflow
launch task, so a
retry only re-polls the terminal sub-run. Use ``max_retries`` to retry
Databricks work inside a
@@ -71,3 +73,54 @@ attempt is observed, so downstream failure handling is not
delayed by long-runni
Only unlimited retries (``max_retries=-1``) keep the Airflow task waiting (or
deferring) until the
parent workflow run reaches a terminal state, because Databricks may still
launch a retry attempt
under the same ``task_key`` until then. Sibling tasks in the run continue
independently.
+
+
+Durable execution
+-----------------
+
+Outside a
+:class:`~airflow.providers.databricks.operators.databricks_workflow.DatabricksWorkflowTaskGroup`,
+this operator submits a run and then waits for it, on the worker or in the
Triggerer when
+``deferrable=True``. By default it runs in a *durable* mode: the Databricks
run id is persisted to
+:doc:`task state store <apache-airflow:core-concepts/task-state-store>` before
the wait begins, so if
+the worker crashes or is preempted and the task is retried, the operator
reconnects to the run that
+is already executing on Databricks instead of submitting a duplicate.
+
+On retry the operator checks the prior run's state:
+
+* if it is still running, the operator reconnects and continues polling
+* if it already succeeded, the operator returns immediately without
resubmitting
+* if it failed terminally, the operator submits a fresh run
+
+Pass ``durable=False`` to always submit a fresh run on retry.
+
+Unlike
:class:`~airflow.providers.databricks.operators.databricks.DatabricksSubmitRunOperator`
and
+:class:`~airflow.providers.databricks.operators.databricks.DatabricksRunNowOperator`,
where
+``durable`` has no effect in deferrable mode, durable execution here also
covers
+``deferrable=True``: the run id is stored before the task defers, so a try
lost while the Triggerer
+was waiting still reconnects on the next one.
+
+Durable execution requires Airflow 3.3 or newer, since it relies on the task
state store. On
+earlier Airflow versions the flag is a no-op and the operator always submits a
fresh run on retry.
+If the task state store is unavailable at runtime, the operator logs that
crash recovery is
+disabled and behaves the same way.
+
+Inside a ``DatabricksWorkflowTaskGroup`` nothing changes: the run is submitted
by the workflow
+launch task and this operator only monitors its own task within that shared
run, so a retry already
+reconnects rather than resubmitting.
+
+``[state_store] default_retention_days`` (30 days by default) is stamped on
the run id when it is
+stored, so a retry that comes later than that finds nothing and submits a
fresh run instead of
+reconnecting, whatever schedule ``airflow state-store clean`` runs on. Size
that setting against
+your longest ``retry_delay``. See
+:doc:`apache-airflow:administration-and-deployment/task-and-asset-state-store-cleanup`.
+
+Clearing a task discards the stored run id by default, so the next attempt
submits a fresh run.
+Set ``keep_task_state`` when clearing to keep it instead.
+:doc:`apache-airflow:core-concepts/resumable-tasks` lists which clear paths
discard it and
+describes the ``[state_store] clear_on_success`` setting.
+
+Where the id is kept, clearing behaves like a retry rather than a fresh start.
If the stored run
+already succeeded, the next attempt reads it back, re-pushes its XComs and
returns without
+submitting anything. If the run is still going and
``wait_for_termination=False``, the attempt
+reconnects to it and returns immediately, also without submitting.
diff --git
a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
index e864881492a..b8a98ac09d6 100644
---
a/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
+++
b/providers/databricks/src/airflow/providers/databricks/operators/databricks.py
@@ -83,6 +83,71 @@ XCOM_JOB_ID_KEY = "job_id"
XCOM_RUN_PAGE_URL_KEY = "run_page_url"
XCOM_STATEMENT_ID_KEY = "statement_id"
+# Databricks splits run state across life_cycle_state and result_state, while
the ResumableJobMixin
+# status interface is a single string. The durable operators encode both as
"LIFE_CYCLE:RESULT" and
+# decode them with the predicates below, so all three share one view of which
runs can be rejoined.
+RUN_NOT_FOUND_STATUS = "NOT_FOUND:"
+TERMINATING_SETTLE_TIMEOUT_SECONDS = 60
+MIN_SETTLE_POLL_SECONDS = 0.1
+ACTIVE_RUN_LIFE_CYCLE_STATES = (
+ "PENDING",
+ "QUEUED",
+ "RUNNING",
+ "TERMINATING",
+ "BLOCKED",
+ "WAITING_FOR_RETRY",
+)
+
+
+def fetch_run_status(
+ hook: DatabricksHook,
+ run_id: JsonValue,
+ *,
+ poll_interval: float = 5,
+ settle_timeout: float = TERMINATING_SETTLE_TIMEOUT_SECONDS,
+) -> str:
+ """
+ Return the encoded status of ``run_id``, or a sentinel if its run history
is gone.
+
+ ``TERMINATING`` is waited out because it does not say whether the run is
finishing on its own or
+ being cancelled, and ``on_kill`` cancels asynchronously without clearing
the stored id. On
+ timeout it is reported as-is, which counts as active: rejoining a slow run
beats duplicating it.
+ """
+ deadline = time.monotonic() + settle_timeout
+ while True:
+ try:
+ # The stored external id is always a Databricks run id; the mixin
types it as JsonValue.
+ run_info = hook.get_run(cast("int", run_id))
+ except DatabricksApiError as e:
+ # The sentinel is neither active nor succeeded, so the mixin
resubmits fresh instead of
+ # failing the task.
+ if _is_run_missing_error(e):
+ hook.log.warning(
+ "Databricks has no record of run %s, so a fresh run will
be submitted. "
+ "If the workspace connection changed, the original run may
still be going there. "
+ "Databricks said: %s",
+ run_id,
+ e,
+ )
+ return RUN_NOT_FOUND_STATUS
+ raise
+ state = RunState(**run_info["state"])
+ remaining = deadline - time.monotonic()
+ if state.life_cycle_state != "TERMINATING" or remaining <= 0:
+ return f"{state.life_cycle_state}:{state.result_state}"
+ time.sleep(max(MIN_SETTLE_POLL_SECONDS, min(poll_interval, remaining)))
+
+
+def is_run_active(status: str) -> bool:
+ """Whether an encoded status means the run is still going and can be
reconnected to."""
+ return status.split(":", 1)[0] in ACTIVE_RUN_LIFE_CYCLE_STATES
+
+
+def is_run_succeeded(status: str) -> bool:
+ """Whether an encoded status means the run finished successfully."""
+ return status == "TERMINATED:SUCCESS"
+
+
# jobs/runs/get answers 400 INVALID_PARAMETER_VALUE for a run it no longer
knows about, never 404.
# These are the two message forms the Databricks SDK overrides to
ResourceDoesNotExist; the code
# alone also covers malformed input, which must stay an error rather than
trigger a resubmit.
@@ -1033,39 +1098,13 @@ class DatabricksSubmitRunOperator(ResumableJobMixin,
BaseOperator):
return self.run_id
def get_job_status(self, external_id: JsonValue, context: Context) -> str:
- # Databricks splits run state across life_cycle_state and
result_state; the mixin's status
- # interface is a single string, so encode both as "LIFE_CYCLE:RESULT"
and decode them in
- # is_job_active / is_job_succeeded.
- try:
- run_info = self._hook.get_run(external_id)
- except DatabricksApiError as e:
- # Neither active nor succeeded, so the mixin resubmits instead of
failing every retry.
- if _is_run_missing_error(e):
- self.log.warning(
- "Databricks has no record of run %s, so a fresh run will
be submitted. "
- "If the workspace connection changed, the original run may
still be going there. "
- "Databricks said: %s",
- external_id,
- e,
- )
- return "NOT_FOUND:"
- raise
- state = RunState(**run_info["state"])
- return f"{state.life_cycle_state}:{state.result_state}"
+ return fetch_run_status(self._hook, external_id,
poll_interval=self.polling_period_seconds)
def is_job_active(self, status: str) -> bool:
- life_cycle_state = status.split(":", 1)[0]
- return life_cycle_state in (
- "PENDING",
- "QUEUED",
- "RUNNING",
- "TERMINATING",
- "BLOCKED",
- "WAITING_FOR_RETRY",
- )
+ return is_run_active(status)
def is_job_succeeded(self, status: str) -> bool:
- return status == "TERMINATED:SUCCESS"
+ return is_run_succeeded(status)
def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
# submit_job and the stored external id are always Databricks run ids
(int).
@@ -1535,39 +1574,13 @@ class DatabricksRunNowOperator(ResumableJobMixin,
BaseOperator):
return self.run_id
def get_job_status(self, external_id: JsonValue, context: Context) -> str:
- # Databricks splits run state across life_cycle_state and
result_state; the mixin's status
- # interface is a single string, so encode both as "LIFE_CYCLE:RESULT"
and decode them in
- # is_job_active / is_job_succeeded.
- try:
- run_info = self._hook.get_run(external_id)
- except DatabricksApiError as e:
- # Neither active nor succeeded, so the mixin resubmits instead of
failing every retry.
- if _is_run_missing_error(e):
- self.log.warning(
- "Databricks has no record of run %s, so a fresh run will
be submitted. "
- "If the workspace connection changed, the original run may
still be going there. "
- "Databricks said: %s",
- external_id,
- e,
- )
- return "NOT_FOUND:"
- raise
- state = RunState(**run_info["state"])
- return f"{state.life_cycle_state}:{state.result_state}"
+ return fetch_run_status(self._hook, external_id,
poll_interval=self.polling_period_seconds)
def is_job_active(self, status: str) -> bool:
- life_cycle_state = status.split(":", 1)[0]
- return life_cycle_state in (
- "PENDING",
- "QUEUED",
- "RUNNING",
- "TERMINATING",
- "BLOCKED",
- "WAITING_FOR_RETRY",
- )
+ return is_run_active(status)
def is_job_succeeded(self, status: str) -> bool:
- return status == "TERMINATED:SUCCESS"
+ return is_run_succeeded(status)
def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
# submit_job and the stored external id are always Databricks run ids
(int).
@@ -1821,7 +1834,7 @@ class
DatabricksSQLStatementsOperator(DatabricksSQLStatementsMixin, BaseOperator
)
-class DatabricksTaskBaseOperator(BaseOperator, ABC):
+class DatabricksTaskBaseOperator(ResumableJobMixin, BaseOperator, ABC):
"""
Base class for operators that are run as Databricks job tasks or tasks
within a Databricks workflow.
@@ -1847,8 +1860,12 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
:param max_retries: Databricks task retry count. Use ``-1`` for unlimited
retries.
:param min_retry_interval_millis: Minimum interval between Databricks task
retries.
:param retry_on_timeout: Whether Databricks retries timed-out tasks.
+ :param durable: Whether to reconnect to an already-submitted run on retry
instead of
+ submitting a new one. Only applies outside a
``DatabricksWorkflowTaskGroup``.
"""
+ external_id_key = "databricks_task_run_id"
+
template_fields: Sequence[str] = ("databricks_conn_id",)
def __init__(
@@ -1869,8 +1886,13 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
max_retries: int | str | None = None,
min_retry_interval_millis: int | str | None = None,
retry_on_timeout: bool | str | None = None,
+ durable: bool | None = None,
**kwargs: Any,
):
+ # Named here (not left to **kwargs) so default_args reaches it on every
+ # supported Airflow version.
+ if durable is not None:
+ kwargs["durable"] = durable
self.caller = caller
self.databricks_conn_id = databricks_conn_id
self._databricks_task_key = databricks_task_key
@@ -1889,6 +1911,7 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
self.retry_on_timeout = retry_on_timeout
self.databricks_run_id: int | None = None
+ self._run_xcoms_pushed = False
super().__init__(**kwargs)
@@ -2047,15 +2070,24 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
def _launch_job(self, context: Context | None = None) -> int | None:
"""Launch the job on Databricks."""
run_json = self._get_run_json()
+ # Return as soon as the run exists. A page URL lookup or an xcom push
here would run
+ # before the mixin can persist the run id, so a failure would strand
the run and let the
+ # retry submit a duplicate.
self.databricks_run_id = self._hook.submit_run(run_json)
+ self.log.info("Run submitted with run_id: %s", self.databricks_run_id)
+ return self.databricks_run_id
+
+ def _record_run_details(self, context: Context) -> None:
+ """Log the run page URL and push the run xcoms, at most once per
attempt."""
+ if self._run_xcoms_pushed or self.databricks_run_id is None:
+ return
url = self._hook.get_run_page_url(self.databricks_run_id)
self.log.info("Check the job run in Databricks: %s", url)
-
- if self.do_xcom_push and context is not None:
- context["ti"].xcom_push(key=XCOM_RUN_ID_KEY,
value=self.databricks_run_id)
- context["ti"].xcom_push(key=XCOM_RUN_PAGE_URL_KEY, value=url)
-
- return self.databricks_run_id
+ ti = context.get("ti") if context else None
+ if self.do_xcom_push and ti is not None:
+ ti.xcom_push(key=XCOM_RUN_ID_KEY, value=self.databricks_run_id)
+ ti.xcom_push(key=XCOM_RUN_PAGE_URL_KEY, value=url)
+ self._run_xcoms_pushed = True
def _handle_terminal_run_state(self, run_state: RunState, errors: list) ->
None:
"""Handle the terminal state of the run."""
@@ -2302,6 +2334,32 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
errors = extract_failed_task_errors(self._hook, run, run_state)
self._handle_terminal_run_state(run_state, errors)
+ def submit_job(self, context: Context) -> int | None:
+ return self._launch_job(context=context)
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ self.databricks_run_id = cast("int", external_id)
+ return fetch_run_status(self._hook, external_id,
poll_interval=self.polling_period_seconds)
+
+ def is_job_active(self, status: str) -> bool:
+ return is_run_active(status)
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return is_run_succeeded(status)
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ self.databricks_run_id = cast("int", external_id)
+ # The mixin's reconnect branch returns from here without reaching
get_job_result, so the
+ # xcoms must be pushed before monitoring or a crash-recovered attempt
leaves them unset.
+ self._record_run_details(context)
+ if self.wait_for_termination:
+ self.monitor_databricks_job()
+
+ def get_job_result(self, external_id: JsonValue, context: Context) -> None:
+ self.databricks_run_id = cast("int", external_id)
+ self._record_run_details(context)
+ return None
+
def execute(self, context: Context) -> None:
"""Execute the operator. Launch the job and monitor it if
wait_for_termination is set to True."""
if self._databricks_workflow_task_group:
@@ -2321,10 +2379,13 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
metadata=workflow_run_metadata,
logger=self.log,
)
- else:
- self._launch_job(context=context)
- if self.wait_for_termination:
- self.monitor_databricks_job()
+ # The run id comes from the upstream launch task's xcom, so a
retry already reconnects
+ # to the shared run rather than submitting anything.
+ if self.wait_for_termination:
+ self.monitor_databricks_job()
+ return
+
+ self.execute_resumable(context)
def execute_complete(self, context: dict | None, event: dict) -> None:
run_state = RunState.from_json(event["run_state"])
@@ -2394,6 +2455,8 @@ class
DatabricksNotebookOperator(DatabricksTaskBaseOperator):
:param max_retries: Databricks task retry count. Use ``-1`` for unlimited
retries.
:param min_retry_interval_millis: Minimum interval between Databricks task
retries.
:param retry_on_timeout: Whether Databricks retries timed-out tasks.
+ :param durable: Whether to reconnect to an already-submitted run on retry
instead of
+ submitting a new one. Only applies outside a
``DatabricksWorkflowTaskGroup``.
"""
template_fields = (
@@ -2426,6 +2489,7 @@ class
DatabricksNotebookOperator(DatabricksTaskBaseOperator):
max_retries: int | str | None = None,
min_retry_interval_millis: int | str | None = None,
retry_on_timeout: bool | str | None = None,
+ durable: bool | None = None,
**kwargs: Any,
):
self.notebook_path = notebook_path
@@ -2449,6 +2513,7 @@ class
DatabricksNotebookOperator(DatabricksTaskBaseOperator):
max_retries=max_retries,
min_retry_interval_millis=min_retry_interval_millis,
retry_on_timeout=retry_on_timeout,
+ durable=durable,
**kwargs,
)
@@ -2550,6 +2615,8 @@ class DatabricksTaskOperator(DatabricksTaskBaseOperator):
:param max_retries: Databricks task retry count, overriding
``task_config`` when set.
:param min_retry_interval_millis: Minimum retry interval, overriding
``task_config`` when set.
:param retry_on_timeout: Whether Databricks retries timed-out tasks,
overriding ``task_config`` when set.
+ :param durable: Whether to reconnect to an already-submitted run on retry
instead of
+ submitting a new one. Only applies outside a
``DatabricksWorkflowTaskGroup``.
"""
CALLER = "DatabricksTaskOperator"
@@ -2579,6 +2646,7 @@ class DatabricksTaskOperator(DatabricksTaskBaseOperator):
max_retries: int | str | None = None,
min_retry_interval_millis: int | str | None = None,
retry_on_timeout: bool | str | None = None,
+ durable: bool | None = None,
**kwargs,
):
self.task_config = task_config
@@ -2599,6 +2667,7 @@ class DatabricksTaskOperator(DatabricksTaskBaseOperator):
max_retries=max_retries,
min_retry_interval_millis=min_retry_interval_millis,
retry_on_timeout=retry_on_timeout,
+ durable=durable,
**kwargs,
)
diff --git
a/providers/databricks/tests/unit/databricks/operators/test_databricks.py
b/providers/databricks/tests/unit/databricks/operators/test_databricks.py
index b933eeade54..f9a7e900ef9 100644
--- a/providers/databricks/tests/unit/databricks/operators/test_databricks.py
+++ b/providers/databricks/tests/unit/databricks/operators/test_databricks.py
@@ -41,6 +41,7 @@ from airflow.providers.common.compat.sdk import
AirflowException, BaseOperator,
from airflow.providers.databricks.exceptions import DatabricksApiError
from airflow.providers.databricks.hooks.databricks import RunState,
SQLStatementState
from airflow.providers.databricks.operators.databricks import (
+ MIN_SETTLE_POLL_SECONDS,
DatabricksCreateJobsOperator,
DatabricksNotebookOperator,
DatabricksRunNowOperator,
@@ -49,6 +50,9 @@ from airflow.providers.databricks.operators.databricks import
(
DatabricksTaskBaseOperator,
DatabricksTaskOperator,
_is_run_missing_error,
+ fetch_run_status,
+ is_run_active,
+ is_run_succeeded,
)
from airflow.providers.databricks.triggers.databricks import (
DatabricksExecutionTrigger,
@@ -331,6 +335,11 @@ def mock_dict(d: dict):
return m
+def make_run_state(lifecycle: str, result: str = "") -> dict:
+ """Minimal ``runs/get`` payload carrying only the run state."""
+ return {"state": {"life_cycle_state": lifecycle, "result_state": result,
"state_message": ""}}
+
+
def make_run_with_state_mock(
lifecycle_state: str, result_state: str, state_message: str = "",
run_id=1, job_id=JOB_ID
):
@@ -2137,10 +2146,6 @@ class TestDatabricksSubmitRunOperatorDurable:
ctx["task_state_store"] = task_store
return ctx
- @staticmethod
- def _state(lifecycle: str, result: str = ""):
- return {"state": {"life_cycle_state": lifecycle, "result_state":
result, "state_message": ""}}
-
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
def test_persists_run_id_to_task_state_store_on_fresh_submit(self,
db_mock_class):
op = DatabricksSubmitRunOperator(
@@ -2165,7 +2170,7 @@ class TestDatabricksSubmitRunOperatorDurable:
)
db_mock = db_mock_class.return_value
# status check sees the stored run still RUNNING, the poll then sees
it finish.
- db_mock.get_run.side_effect = [self._state("RUNNING"),
self._state("TERMINATED", "SUCCESS")]
+ db_mock.get_run.side_effect = [make_run_state("RUNNING"),
make_run_state("TERMINATED", "SUCCESS")]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = RUN_ID
@@ -2182,7 +2187,7 @@ class TestDatabricksSubmitRunOperatorDurable:
)
db_mock = db_mock_class.return_value
# A gated run sits in BLOCKED; the durable retry must reconnect and
wait, not resubmit.
- db_mock.get_run.side_effect = [self._state("BLOCKED"),
self._state("TERMINATED", "SUCCESS")]
+ db_mock.get_run.side_effect = [make_run_state("BLOCKED"),
make_run_state("TERMINATED", "SUCCESS")]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = RUN_ID
@@ -2244,8 +2249,8 @@ class TestDatabricksSubmitRunOperatorDurable:
db_mock.submit_run.return_value = RUN_ID
# stored run failed, after fresh resubmit the poll sees success.
db_mock.get_run.side_effect = [
- self._state("TERMINATED", "FAILED"),
- self._state("TERMINATED", "SUCCESS"),
+ make_run_state("TERMINATED", "FAILED"),
+ make_run_state("TERMINATED", "SUCCESS"),
]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = 999
@@ -2265,7 +2270,7 @@ class TestDatabricksSubmitRunOperatorDurable:
db_mock.submit_run.return_value = RUN_ID
db_mock.get_run.side_effect = [
make_api_error(400, "INVALID_PARAMETER_VALUE", "Run 999 does not
exist."),
- self._state("TERMINATED", "SUCCESS"),
+ make_run_state("TERMINATED", "SUCCESS"),
]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = 999
@@ -2326,7 +2331,7 @@ class TestDatabricksSubmitRunOperatorDurable:
def test_get_job_status_encodes_life_cycle_and_result(self, db_mock_class):
op = DatabricksSubmitRunOperator(task_id=TASK_ID,
json={"notebook_task": NOTEBOOK_TASK})
db_mock = db_mock_class.return_value
- db_mock.get_run.return_value = self._state("TERMINATED", "SUCCESS")
+ db_mock.get_run.return_value = make_run_state("TERMINATED", "SUCCESS")
assert op.get_job_status(RUN_ID, {}) == "TERMINATED:SUCCESS"
@@ -3524,10 +3529,6 @@ class TestDatabricksRunNowOperatorDurable:
ctx["task_state_store"] = task_store
return ctx
- @staticmethod
- def _state(lifecycle: str, result: str = ""):
- return {"state": {"life_cycle_state": lifecycle, "result_state":
result, "state_message": ""}}
-
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
def test_persists_run_id_to_task_state_store_on_fresh_submit(self,
db_mock_class):
op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID)
@@ -3548,7 +3549,7 @@ class TestDatabricksRunNowOperatorDurable:
op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID)
db_mock = db_mock_class.return_value
# status check sees the stored run still RUNNING, the poll then sees
it finish.
- db_mock.get_run.side_effect = [self._state("RUNNING"),
self._state("TERMINATED", "SUCCESS")]
+ db_mock.get_run.side_effect = [make_run_state("RUNNING"),
make_run_state("TERMINATED", "SUCCESS")]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = RUN_ID
@@ -3563,7 +3564,7 @@ class TestDatabricksRunNowOperatorDurable:
op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID)
db_mock = db_mock_class.return_value
# A gated run sits in BLOCKED; the durable retry must reconnect and
wait, not resubmit.
- db_mock.get_run.side_effect = [self._state("BLOCKED"),
self._state("TERMINATED", "SUCCESS")]
+ db_mock.get_run.side_effect = [make_run_state("BLOCKED"),
make_run_state("TERMINATED", "SUCCESS")]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = RUN_ID
@@ -3578,7 +3579,7 @@ class TestDatabricksRunNowOperatorDurable:
op = DatabricksRunNowOperator(task_id=TASK_ID, job_name=JOB_NAME)
db_mock = db_mock_class.return_value
db_mock.find_job_id_by_name.return_value = JOB_ID
- db_mock.get_run.side_effect = [self._state("RUNNING"),
self._state("TERMINATED", "SUCCESS")]
+ db_mock.get_run.side_effect = [make_run_state("RUNNING"),
make_run_state("TERMINATED", "SUCCESS")]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = RUN_ID
@@ -3635,8 +3636,8 @@ class TestDatabricksRunNowOperatorDurable:
db_mock = db_mock_class.return_value
db_mock.run_now.return_value = RUN_ID
db_mock.get_run.side_effect = [
- self._state("TERMINATED", "FAILED"),
- self._state("TERMINATED", "SUCCESS"),
+ make_run_state("TERMINATED", "FAILED"),
+ make_run_state("TERMINATED", "SUCCESS"),
]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = 999
@@ -3655,7 +3656,7 @@ class TestDatabricksRunNowOperatorDurable:
# the stored run's history has expired -> get_run rejects it; the
fresh run polls to success.
db_mock.get_run.side_effect = [
make_api_error(400, "INVALID_PARAMETER_VALUE", "Run 999 does not
exist."),
- self._state("TERMINATED", "SUCCESS"),
+ make_run_state("TERMINATED", "SUCCESS"),
]
task_store = MagicMock(spec_set=["get", "set"])
task_store.get.return_value = 999
@@ -3711,7 +3712,7 @@ class TestDatabricksRunNowOperatorDurable:
def test_get_job_status_encodes_life_cycle_and_result(self, db_mock_class):
op = DatabricksRunNowOperator(task_id=TASK_ID, job_id=JOB_ID)
db_mock = db_mock_class.return_value
- db_mock.get_run.return_value = self._state("TERMINATED", "SUCCESS")
+ db_mock.get_run.return_value = make_run_state("TERMINATED", "SUCCESS")
assert op.get_job_status(RUN_ID, {}) == "TERMINATED:SUCCESS"
@@ -4179,7 +4180,8 @@ class TestDatabricksNotebookOperator:
assert isinstance(operator, DatabricksTaskBaseOperator)
- def test_execute_with_wait_for_termination(self):
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_execute_with_wait_for_termination(self, _):
operator = DatabricksNotebookOperator(
task_id="test_task",
notebook_path="test_path",
@@ -4195,7 +4197,8 @@ class TestDatabricksNotebookOperator:
operator._launch_job.assert_called_once()
operator.monitor_databricks_job.assert_called_once()
- def test_execute_without_wait_for_termination(self):
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_execute_without_wait_for_termination(self, _):
operator = DatabricksNotebookOperator(
task_id="test_task",
notebook_path="test_path",
@@ -5350,3 +5353,328 @@ class TestDatabricksTaskOperator:
operator.monitor_databricks_job()
assert mock_databricks_hook.return_value.get_run.call_count == 3
+
+
[email protected](
+ not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution)
requires Airflow 3.3+"
+)
+class TestDatabricksTaskBaseOperatorDurable:
+ @pytest.fixture(params=["notebook", "task"])
+ def make_operator(self, request):
+ def _make(**kwargs):
+ if request.param == "notebook":
+ return DatabricksNotebookOperator(
+ task_id=TASK_ID,
+ notebook_path="/test",
+ source="WORKSPACE",
+ new_cluster=NEW_CLUSTER,
+ **kwargs,
+ )
+ return DatabricksTaskOperator(
+ task_id=TASK_ID,
+ task_config={"notebook_task": NOTEBOOK_TASK},
+ new_cluster=NEW_CLUSTER,
+ **kwargs,
+ )
+
+ return _make
+
+ @staticmethod
+ def _task_store(stored_run_id=None):
+ task_store = MagicMock(spec_set=["get", "set"])
+ task_store.get.return_value = stored_run_id
+ return task_store
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_persists_run_id_on_fresh_submit(self, db_mock_class,
make_operator):
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run_page_url.return_value = RUN_PAGE_URL
+ task_store = self._task_store()
+ ti = MagicMock(stats_tags={})
+
+ op.execute({"ti": ti, "task_state_store": task_store})
+
+ task_store.set.assert_called_once_with("databricks_task_run_id",
RUN_ID)
+ op.monitor_databricks_job.assert_called_once()
+ assert ti.xcom_push.call_count == 2
+ ti.xcom_push.assert_any_call(key="run_id", value=RUN_ID)
+ ti.xcom_push.assert_any_call(key="run_page_url", value=RUN_PAGE_URL)
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_persists_run_id_even_if_the_page_url_lookup_fails(self,
db_mock_class, make_operator):
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run_page_url.side_effect = DatabricksApiError("boom",
http_status_code=500)
+ task_store = self._task_store()
+
+ with pytest.raises(DatabricksApiError):
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ task_store.set.assert_called_once_with("databricks_task_run_id",
RUN_ID)
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_reconnects_to_active_run_without_resubmitting(self,
db_mock_class, make_operator):
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.get_run.return_value = make_run_state("RUNNING")
+ db_mock.get_run_page_url.return_value = RUN_PAGE_URL
+ task_store = self._task_store(RUN_ID)
+ ti = MagicMock(stats_tags={})
+
+ op.execute({"ti": ti, "task_state_store": task_store})
+
+ db_mock.submit_run.assert_not_called()
+ task_store.set.assert_not_called()
+ op.monitor_databricks_job.assert_called_once()
+ assert op.databricks_run_id == RUN_ID
+ # A crash-recovered attempt still has to publish the run link
downstream.
+ ti.xcom_push.assert_any_call(key="run_id", value=RUN_ID)
+ ti.xcom_push.assert_any_call(key="run_page_url", value=RUN_PAGE_URL)
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_already_succeeded_pushes_xcoms_without_monitoring(self,
db_mock_class, make_operator):
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.get_run.return_value = make_run_state("TERMINATED", "SUCCESS")
+ db_mock.get_run_page_url.return_value = RUN_PAGE_URL
+ ti = MagicMock(stats_tags={})
+
+ op.execute({"ti": ti, "task_state_store": self._task_store(RUN_ID)})
+
+ db_mock.submit_run.assert_not_called()
+ op.monitor_databricks_job.assert_not_called()
+ ti.xcom_push.assert_any_call(key="run_id", value=RUN_ID)
+ ti.xcom_push.assert_any_call(key="run_page_url", value=RUN_PAGE_URL)
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_resubmits_when_stored_run_in_terminal_failure(self,
db_mock_class, make_operator):
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run.return_value = make_run_state("TERMINATED", "FAILED")
+ task_store = self._task_store(999)
+
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_task_run_id",
RUN_ID)
+ assert op.databricks_run_id == RUN_ID
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_resubmits_when_stored_run_no_longer_exists(self, db_mock_class,
make_operator):
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run.side_effect = make_api_error(
+ 400, "INVALID_PARAMETER_VALUE", "Run 999 does not exist."
+ )
+ task_store = self._task_store(999)
+
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_task_run_id",
RUN_ID)
+
+ @mock.patch("airflow.providers.databricks.operators.databricks.time.sleep")
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_resubmits_when_stored_run_was_cancelled(self, db_mock_class, _,
make_operator):
+ """Clearing a deferred task cancels the run via the trigger but leaves
its id stored."""
+ op = make_operator()
+ op.monitor_databricks_job = MagicMock()
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run.side_effect = [
+ make_run_state("TERMINATING", ""),
+ make_run_state("TERMINATED", "CANCELED"),
+ ]
+ task_store = self._task_store(999)
+
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ db_mock.submit_run.assert_called_once()
+ task_store.set.assert_called_once_with("databricks_task_run_id",
RUN_ID)
+
+ @mock.patch(
+
"airflow.providers.databricks.operators.databricks.DatabricksTaskBaseOperator._get_current_databricks_task"
+ )
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_stores_run_id_before_deferring(self, db_mock_class,
mock_current_task, make_operator):
+ op = make_operator(deferrable=True)
+ db_mock = db_mock_class.return_value
+ db_mock.submit_run.return_value = RUN_ID
+ db_mock.get_run.return_value = {
+ "state": {"life_cycle_state": "PENDING", "result_state": ""},
+ "run_page_url": "test_url",
+ }
+ mock_current_task.return_value = {"run_id": RUN_ID}
+ task_store = self._task_store()
+ stored_when_deferred = []
+
+ def record_then_defer(*args, **kwargs):
+ stored_when_deferred.append(task_store.set.call_args)
+ raise TaskDeferred(trigger=MagicMock(),
method_name="execute_complete")
+
+ op.defer = record_then_defer
+
+ with pytest.raises(TaskDeferred):
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ assert stored_when_deferred == [mock.call("databricks_task_run_id",
RUN_ID)]
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_durable_false_never_touches_task_state_store(self, db_mock_class,
make_operator):
+ op = make_operator(durable=False)
+ op.monitor_databricks_job = MagicMock()
+ db_mock_class.return_value.submit_run.return_value = RUN_ID
+ task_store = self._task_store()
+
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ db_mock_class.return_value.submit_run.assert_called_once()
+ task_store.get.assert_not_called()
+ task_store.set.assert_not_called()
+
+ def test_default_args_durable_reaches_operator(self, make_operator):
+ assert make_operator(default_args={"durable": False}).durable is False
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.store_databricks_job_run_link")
+ @mock.patch(
+
"airflow.providers.databricks.operators.databricks.DatabricksTaskBaseOperator._databricks_workflow_task_group",
+ new_callable=mock.PropertyMock,
+ )
+
@mock.patch("airflow.providers.databricks.operators.databricks.DatabricksHook")
+ def test_workflow_task_group_does_not_use_task_state_store(
+ self, db_mock_class, mock_tg, _, make_operator
+ ):
+ mock_tg.return_value = MagicMock()
+ op = make_operator(
+ workflow_run_metadata={"conn_id": "databricks_default", "job_id":
1, "run_id": RUN_ID}
+ )
+ op.monitor_databricks_job = MagicMock()
+ task_store = self._task_store()
+
+ op.execute({"ti": MagicMock(stats_tags={}), "task_state_store":
task_store})
+
+ task_store.get.assert_not_called()
+ task_store.set.assert_not_called()
+ db_mock_class.return_value.submit_run.assert_not_called()
+ op.monitor_databricks_job.assert_called_once()
+ assert op.databricks_run_id == RUN_ID
+
+
[email protected](
+ not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution)
requires Airflow 3.3+"
+)
+class TestRunStatusHelpers:
+ """The encoding and predicates shared by all three durable Databricks
operators."""
+
+ @pytest.mark.parametrize(
+ ("status", "expected"),
+ [
+ ("PENDING:", True),
+ ("QUEUED:", True),
+ ("RUNNING:", True),
+ ("TERMINATING:", True),
+ ("BLOCKED:", True),
+ ("WAITING_FOR_RETRY:", True),
+ ("TERMINATED:SUCCESS", False),
+ ("TERMINATED:FAILED", False),
+ ("SKIPPED:", False),
+ ("INTERNAL_ERROR:", False),
+ ("NOT_FOUND:", False),
+ ],
+ )
+ def test_is_run_active(self, status, expected):
+ assert is_run_active(status) is expected
+
+ @pytest.mark.parametrize(
+ ("status", "expected"),
+ [
+ ("TERMINATED:SUCCESS", True),
+ ("TERMINATED:FAILED", False),
+ ("RUNNING:", False),
+ ("NOT_FOUND:", False),
+ ],
+ )
+ def test_is_run_succeeded(self, status, expected):
+ assert is_run_succeeded(status) is expected
+
+ def test_fetch_run_status_encodes_life_cycle_and_result(self):
+ hook = MagicMock()
+ hook.get_run.return_value = make_run_state("TERMINATED", "SUCCESS")
+
+ assert fetch_run_status(hook, RUN_ID) == "TERMINATED:SUCCESS"
+
+ def test_fetch_run_status_reports_sentinel_for_missing_run(self):
+ hook = MagicMock()
+ hook.get_run.side_effect = make_api_error(400,
"INVALID_PARAMETER_VALUE", "Run 999 does not exist.")
+
+ assert fetch_run_status(hook, RUN_ID) == "NOT_FOUND:"
+
+ def test_fetch_run_status_reraises_other_errors(self):
+ hook = MagicMock()
+ hook.get_run.side_effect = DatabricksApiError("boom",
http_status_code=500)
+
+ with pytest.raises(DatabricksApiError):
+ fetch_run_status(hook, RUN_ID)
+
+ @pytest.mark.parametrize(
+ ("final_result", "expected"),
+ [("CANCELED", "TERMINATED:CANCELED"), ("SUCCESS",
"TERMINATED:SUCCESS")],
+ )
+ @mock.patch("airflow.providers.databricks.operators.databricks.time.sleep")
+ def test_fetch_run_status_waits_out_terminating(self, mock_sleep,
final_result, expected):
+ hook = MagicMock()
+ hook.get_run.side_effect = [
+ make_run_state("TERMINATING", ""),
+ make_run_state("TERMINATING", ""),
+ make_run_state("TERMINATED", final_result),
+ ]
+
+ assert fetch_run_status(hook, RUN_ID, poll_interval=7) == expected
+ assert hook.get_run.call_count == 3
+ assert mock_sleep.call_args_list == [mock.call(7), mock.call(7)]
+
+ @mock.patch("airflow.providers.databricks.operators.databricks.time.sleep")
+ def test_fetch_run_status_gives_up_on_a_stuck_terminating_run(self,
mock_sleep):
+ hook = MagicMock()
+ hook.get_run.return_value = make_run_state("TERMINATING", "")
+
+ status = fetch_run_status(hook, RUN_ID, settle_timeout=0)
+
+ assert status == "TERMINATING:"
+ assert is_run_active(status) is True
+ hook.get_run.assert_called_once()
+ mock_sleep.assert_not_called()
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.time.monotonic")
+ @mock.patch("airflow.providers.databricks.operators.databricks.time.sleep")
+ def test_fetch_run_status_never_sleeps_past_the_settle_deadline(self,
mock_sleep, mock_monotonic):
+ mock_monotonic.side_effect = [0.0, 0.5, 1.5]
+ hook = MagicMock()
+ hook.get_run.return_value = make_run_state("TERMINATING", "")
+
+ status = fetch_run_status(hook, RUN_ID, poll_interval=30,
settle_timeout=1)
+
+ assert status == "TERMINATING:"
+ assert mock_sleep.call_args_list == [mock.call(0.5)]
+
+
@mock.patch("airflow.providers.databricks.operators.databricks.time.monotonic")
+ @mock.patch("airflow.providers.databricks.operators.databricks.time.sleep")
+ def test_fetch_run_status_does_not_busy_loop_on_a_zero_poll_interval(self,
mock_sleep, mock_monotonic):
+ mock_monotonic.side_effect = [0.0, 0.5, 1.5]
+ hook = MagicMock()
+ hook.get_run.return_value = make_run_state("TERMINATING", "")
+
+ fetch_run_status(hook, RUN_ID, poll_interval=0, settle_timeout=1)
+
+ assert mock_sleep.call_args_list ==
[mock.call(MIN_SETTLE_POLL_SECONDS)]