This is an automated email from the ASF dual-hosted git repository.
amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6120f7a0c06 Support durable execution for AWS BatchOperator (#77158)
6120f7a0c06 is described below
commit 6120f7a0c0621777888cd2982fb8c92df30f6c7d
Author: Amogh Desai <[email protected]>
AuthorDate: Thu Nov 19 17:27:54 2026 +0530
Support durable execution for AWS BatchOperator (#77158)
---
providers/amazon/docs/changelog.rst | 8 ++
providers/amazon/docs/operators/batch.rst | 26 +++++
.../providers/amazon/aws/operators/batch.py | 62 ++++++++--
.../tests/unit/amazon/aws/operators/test_batch.py | 129 ++++++++++++++++++++-
4 files changed, 214 insertions(+), 11 deletions(-)
diff --git a/providers/amazon/docs/changelog.rst
b/providers/amazon/docs/changelog.rst
index ee50e6561c0..e1cc68887ec 100644
--- a/providers/amazon/docs/changelog.rst
+++ b/providers/amazon/docs/changelog.rst
@@ -29,6 +29,14 @@ Changelog
9.41.0
......
+.. note::
+
+ ``BatchOperator`` supports durable execution and ``durable`` defaults to
``True``. A retry now
+ reconnects to the Batch job the previous try submitted instead of
submitting a new one. This
+ applies to the synchronous path only. Pass ``durable=False`` to keep the
old behaviour.
+ Reconnecting needs Airflow 3.3 or later. On Airflow 3.3 and 3.4 most
clears keep the stored job
+ id, so a cleared task whose job already succeeded returns without
submitting a new one.
+
.. note::
``SageMakerTrainingOperator`` supports durable execution and ``durable``
defaults to ``True``.
diff --git a/providers/amazon/docs/operators/batch.rst
b/providers/amazon/docs/operators/batch.rst
index b992e26e4b0..c4581669c87 100644
--- a/providers/amazon/docs/operators/batch.rst
+++ b/providers/amazon/docs/operators/batch.rst
@@ -51,6 +51,32 @@ use
:class:`~airflow.providers.amazon.aws.operators.batch.BatchOperator`.
:start-after: [START howto_operator_batch]
:end-before: [END howto_operator_batch]
+Durable execution
+=================
+
+By default ``BatchOperator`` saves the Batch job id to the
+:doc:`task state store <apache-airflow:core-concepts/task-state-store>` right
after submitting the
+job. If the worker crashes and the task is retried, the operator checks that
job first:
+
+* still active (``SUBMITTED`` through ``RUNNING``): reconnects and keeps
waiting on it
+* already ``SUCCEEDED``: returns the job id without submitting again
+* ``FAILED``, or no longer returned by AWS Batch: submits a new job
+
+This applies to the synchronous path only; a deferrable run is unchanged.
+
+Reconnecting only helps when the worker dies without a chance to clean up,
such as an OOM kill or a
+lost node. On SIGTERM (for example a pod eviction) or on
``execution_timeout``, ``on_kill`` terminates
+the Batch job, so the retry submits a new one.
+
+Clearing a task can keep the stored job id. On Airflow 3.3 every clear keeps
it. On 3.4 only clearing
+task instances through the UI or REST API discards it. When the id is kept and
that job already
+succeeded, the cleared task returns without submitting a new job. From Airflow
3.5, every clear
+discards it by default. See
:doc:`apache-airflow:core-concepts/resumable-tasks` for the clear paths,
+and set ``durable=False`` if every clear must submit a new job.
+
+Set ``durable=False`` to submit a new job on every try. Durable execution
needs Airflow 3.3 or
+later; on earlier versions the operator always submits a new job.
+
.. _howto/operator:BatchCreateComputeEnvironmentOperator:
Create an AWS Batch compute environment
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
b/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
index 610ab9e5952..7831cd5d09a 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/batch.py
@@ -28,7 +28,7 @@ from __future__ import annotations
from collections.abc import Sequence
from datetime import timedelta
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
from airflow.providers.amazon.aws.links.batch import (
@@ -46,13 +46,15 @@ from airflow.providers.amazon.aws.utils import
trim_none_values, validate_execut
from airflow.providers.amazon.aws.utils.deferral import should_defer
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
from airflow.providers.amazon.aws.utils.task_log_fetcher import
AwsTaskLogFetcher
-from airflow.providers.common.compat.sdk import AirflowException, conf
+from airflow.providers.common.compat.sdk import AirflowException,
ResumableJobMixin, conf
if TYPE_CHECKING:
+ from pydantic import JsonValue
+
from airflow.sdk import Context
-class BatchOperator(AwsBaseOperator[BatchClientHook]):
+class BatchOperator(ResumableJobMixin, AwsBaseOperator[BatchClientHook]):
"""
Execute a job on AWS Batch.
@@ -101,6 +103,8 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
:param awslogs_fetch_interval: The interval with which cloudwatch logs are
to be fetched, 30 sec.
:param poll_interval: (Deferrable mode only) Time in seconds to wait
between polling.
:param submit_job_timeout: Execution timeout in seconds for submitted
batch job.
+ :param durable: If True (default), a retry reconnects to the job submitted
by the previous try
+ instead of submitting a new one. Synchronous mode only; requires
Airflow 3.3+.
.. note::
Any custom waiters must return a waiter for these calls:
@@ -112,6 +116,7 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
"""
aws_hook_class = BatchClientHook
+ external_id_key = "batch_job_id"
ui_color = "#c3dae0"
arn: str | None = None
@@ -175,8 +180,11 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
awslogs_enabled: bool = False,
awslogs_fetch_interval: timedelta = timedelta(seconds=30),
submit_job_timeout: int | None = None,
+ durable: bool | None = None,
**kwargs,
) -> None:
+ if durable is not None:
+ kwargs["durable"] = durable
super().__init__(**kwargs)
self.job_id = job_id
self.job_name = job_name
@@ -220,9 +228,8 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
:raises ValueError: if job_id is not found
:raises AirflowException: if job submission fails or job ends in
failure state
"""
- self.submit_job(context)
-
if should_defer(deferrable=self.deferrable,
wait_for_completion=self.wait_for_completion):
+ self.submit_job(context)
if not self.job_id:
raise ValueError("AWS Batch job - job_id was not found")
@@ -256,9 +263,7 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
raise AirflowException(f"Unexpected status: {job_status}")
- if self.wait_for_completion:
- self.monitor_job(context)
-
+ self.execute_resumable(context)
return self.job_id
def execute_complete(self, context: Context, event: dict[str, Any] | None
= None) -> str:
@@ -302,7 +307,7 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
response = self.hook.client.terminate_job(jobId=self.job_id,
reason="Task killed by the user")
self.log.info("AWS Batch job (%s) terminated: %s", self.job_id,
response)
- def submit_job(self, context: Context):
+ def submit_job(self, context: Context) -> str:
"""
Submit an AWS Batch job.
@@ -356,6 +361,45 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
self.job_id = response["jobId"]
self.log.info("AWS Batch job (%s) started: %s", self.job_id, response)
+ self._persist_job_details_link(context)
+ return self.job_id
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ jobs = self.hook.client.describe_jobs(jobs=[cast("str",
external_id)])["jobs"]
+ return jobs[0]["status"] if jobs else "NOT_FOUND"
+
+ def is_job_active(self, status: str) -> bool:
+ return status in self.hook.INTERMEDIATE_STATES
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return status == self.hook.SUCCESS_STATE
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ self._attach_to_job(external_id, context)
+ if self.wait_for_completion:
+ self.monitor_job(context)
+
+ def get_job_result(self, external_id: JsonValue, context: Context) -> str
| None:
+ # Not attached yet only when the stored job already succeeded, so
monitor_job never ran.
+ if external_id is not None and self.job_id != external_id:
+ self._attach_to_job(external_id, context)
+ self._persist_links(context)
+ self._persist_cloudwatch_link(context)
+ return self.job_id
+
+ def _attach_to_job(self, external_id: JsonValue, context: Context) -> None:
+ # None comes from subclasses whose submit_job override predates it
returning the job id;
+ # they still set self.job_id, so keep it.
+ if external_id is None:
+ if self.durable:
+ self.log.warning("submit_job returned no job id, so a retry
will submit a new job")
+ return
+ if self.job_id == external_id:
+ return
+ self.job_id = cast("str", external_id)
+ self._persist_job_details_link(context)
+
+ def _persist_job_details_link(self, context: Context) -> None:
BatchJobDetailsLink.persist(
context=context,
operator=self,
diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
b/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
index 1cd8dac185e..797c35fbda3 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_batch.py
@@ -33,6 +33,7 @@ from airflow.providers.amazon.aws.triggers.batch import (
)
from airflow.providers.common.compat.sdk import AirflowException, TaskDeferred
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_3_PLUS
from unit.amazon.aws.utils.test_deferral import expect_deferral_warning
from unit.amazon.aws.utils.test_template_fields import validate_template_fields
@@ -91,7 +92,7 @@ class TestBatchOperator:
assert self.batch.job_id is None
self.batch.job_id = JOB_ID
- self.mock_context = mock.MagicMock()
+ self.mock_context = {"ti": mock.MagicMock()}
def test_init(self):
assert self.batch.job_id == JOB_ID
@@ -418,7 +419,7 @@ class TestBatchOperator:
wait_for_completion=False,
)
- batch.execute(None)
+ batch.execute({})
expected_args = {
"jobQueue": "queue",
@@ -730,6 +731,130 @@ class TestBatchOperator:
mock_get_job_all_awslogs_info.assert_called_once_with("12345")
[email protected](not AIRFLOW_V_3_3_PLUS, reason="durable execution requires
Airflow 3.3+")
+class TestBatchOperatorDurable:
+ NEW_JOB_ID = "11111111-2222-3333-4444-555555555555"
+
+ @pytest.fixture
+ def client(self):
+ with
mock.patch("airflow.providers.amazon.aws.hooks.batch_client.AwsBaseHook.get_client_type")
as m:
+ m.return_value.submit_job.return_value = {"jobName": JOB_NAME,
"jobId": self.NEW_JOB_ID}
+ yield m.return_value
+
+ @pytest.fixture
+ def task_store(self):
+ store = mock.MagicMock(spec_set=["get", "set"])
+ store.get.return_value = None
+ return store
+
+ @staticmethod
+ def _make_operator(operator_class=BatchOperator, **kwargs):
+ return operator_class(
+ task_id="task",
+ job_name=JOB_NAME,
+ job_queue="queue",
+ job_definition="hello-world",
+ do_xcom_push=False,
+ **kwargs,
+ )
+
+ @staticmethod
+ def _context(task_store):
+ return {"ti": mock.MagicMock(spec_set=["stats_tags"], stats_tags={}),
"task_state_store": task_store}
+
+
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_fresh_submit_persists_job_id(self, monitor_mock, link_mock,
client, task_store):
+ result = self._make_operator().execute(self._context(task_store))
+
+ client.submit_job.assert_called_once()
+ link_mock.assert_called_once()
+ task_store.set.assert_called_once_with("batch_job_id", self.NEW_JOB_ID)
+ monitor_mock.assert_called_once()
+ assert result == self.NEW_JOB_ID
+
+ @pytest.mark.parametrize("status", ["SUBMITTED", "PENDING", "RUNNABLE",
"STARTING", "RUNNING"])
+
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_reconnects_to_active_job(self, monitor_mock, link_mock, client,
task_store, status):
+ task_store.get.return_value = JOB_ID
+ client.describe_jobs.return_value = {"jobs": [{"jobId": JOB_ID,
"status": status}]}
+ operator = self._make_operator()
+
+ result = operator.execute(self._context(task_store))
+
+ client.describe_jobs.assert_called_once_with(jobs=[JOB_ID])
+ client.submit_job.assert_not_called()
+ task_store.set.assert_not_called()
+ monitor_mock.assert_called_once()
+ assert link_mock.call_args.kwargs["job_id"] == JOB_ID
+ assert operator.job_id == JOB_ID
+ assert result == JOB_ID
+
+ @mock.patch.object(BatchOperator, "_persist_cloudwatch_link",
autospec=True)
+ @mock.patch.object(BatchOperator, "_persist_links", autospec=True)
+
@mock.patch("airflow.providers.amazon.aws.links.batch.BatchJobDetailsLink.persist")
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_already_succeeded_skips_submit_and_wait(
+ self, monitor_mock, link_mock, persist_links_mock,
persist_cloudwatch_mock, client, task_store
+ ):
+ task_store.get.return_value = JOB_ID
+ client.describe_jobs.return_value = {"jobs": [{"jobId": JOB_ID,
"status": "SUCCEEDED"}]}
+
+ result = self._make_operator().execute(self._context(task_store))
+
+ client.submit_job.assert_not_called()
+ monitor_mock.assert_not_called()
+ assert link_mock.call_args.kwargs["job_id"] == JOB_ID
+ persist_links_mock.assert_called_once()
+ persist_cloudwatch_mock.assert_called_once()
+ assert result == JOB_ID
+
+ @pytest.mark.parametrize(
+ "describe_response",
+ [
+ pytest.param({"jobs": [{"jobId": JOB_ID, "status": "FAILED"}]},
id="failed"),
+ pytest.param({"jobs": []}, id="not-found"),
+ ],
+ )
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_resubmits_when_stored_job_not_active(self, monitor_mock, client,
task_store, describe_response):
+ task_store.get.return_value = JOB_ID
+ client.describe_jobs.return_value = describe_response
+
+ result = self._make_operator().execute(self._context(task_store))
+
+ client.submit_job.assert_called_once()
+ task_store.set.assert_called_once_with("batch_job_id", self.NEW_JOB_ID)
+ assert result == self.NEW_JOB_ID
+
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_durable_false_does_not_use_task_state_store(self, monitor_mock,
client, task_store):
+ self._make_operator(durable=False).execute(self._context(task_store))
+
+ client.submit_job.assert_called_once()
+ task_store.get.assert_not_called()
+ task_store.set.assert_not_called()
+
+ @pytest.mark.parametrize("durable", [True, False])
+ @mock.patch.object(BatchOperator, "monitor_job", autospec=True)
+ def test_submit_job_override_returning_none_keeps_job_id(
+ self, monitor_mock, client, task_store, caplog, durable
+ ):
+ class LegacySubmitBatchOperator(BatchOperator):
+ def submit_job(self, context):
+ super().submit_job(context)
+
+ operator =
self._make_operator(operator_class=LegacySubmitBatchOperator, durable=durable)
+
+ result = operator.execute(self._context(task_store))
+
+ monitor_mock.assert_called_once()
+ assert operator.job_id == self.NEW_JOB_ID
+ assert result == self.NEW_JOB_ID
+ assert ("submit_job returned no job id, so a retry will submit a new
job" in caplog) is durable
+
+
class TestBatchCreateComputeEnvironmentOperator:
warn_message = "The `status_retries` parameter is unused and should be
removed"