This is an automated email from the ASF dual-hosted git repository.

amoghrajesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 842402c7eee Test reconnect on_kill and document SageMaker durable 
execution (#76964)
842402c7eee is described below

commit 842402c7eeef1698a19030931832e82a81f87795
Author: Amogh Desai <[email protected]>
AuthorDate: Wed Nov 18 15:36:35 2026 +0530

    Test reconnect on_kill and document SageMaker durable execution (#76964)
---
 providers/amazon/docs/operators/sagemaker.rst      | 61 ++++++++++++++++++++++
 .../providers/amazon/aws/operators/sagemaker.py    | 10 ++--
 .../aws/operators/test_sagemaker_training.py       | 25 ++++++++-
 3 files changed, 89 insertions(+), 7 deletions(-)

diff --git a/providers/amazon/docs/operators/sagemaker.rst 
b/providers/amazon/docs/operators/sagemaker.rst
index e9e703f4f01..89526a81d6a 100644
--- a/providers/amazon/docs/operators/sagemaker.rst
+++ b/providers/amazon/docs/operators/sagemaker.rst
@@ -67,6 +67,67 @@ To create an Amazon Sagemaker training job you can use
     :start-after: [START howto_operator_sagemaker_training]
     :end-before: [END howto_operator_sagemaker_training]
 
+Durable execution
+"""""""""""""""""
+
+``SageMakerTrainingOperator`` submits a training job and then polls it to 
completion on the
+worker. By default the operator runs in a *durable* mode that makes this 
crash-safe: the training
+job name is persisted to :doc:`task state store 
<apache-airflow:core-concepts/task-state-store>`
+before polling begins, so if the worker crashes or is preempted and the task 
is retried, the
+operator reconnects to the job that is already running in SageMaker instead of 
starting a new one.
+
+On retry the operator checks the prior job's status:
+
+* if it is ``InProgress``, the operator reconnects and continues polling
+* if it is ``Completed``, the operator returns its result without resubmitting
+* if it is ``Failed``, ``Stopped`` or ``Stopping``, or the name is no longer 
found, the operator
+  submits a new job
+
+Unlike Glue, a job that is still ``Stopping`` gets a new submit rather than a 
reconnect. With
+``print_log=False``, the wait treats ``Stopped`` as done, so reconnecting 
would let the task succeed
+without a finished training run. ``cancel_on_kill`` leaves the job in exactly 
that state when a
+timed out attempt is killed. A new submit is renamed with a timestamp suffix 
when
+``check_if_job_exists=True`` and ``action_if_job_exists="timestamp"`` (both 
defaults), because the
+earlier job still holds its name. With ``check_if_job_exists=False``, 
SageMaker rejects the new
+submit.
+Spot training resume through ``CheckpointConfig`` is unaffected, since the 
checkpoint
+``S3Uri`` does not depend on the job name.
+
+On Airflow 3.3, clearing a task keeps the stored job name, so the next attempt 
is treated the same
+as a retry. A task whose job already completed returns the existing result 
without starting a new
+job.
+
+From Airflow 3.4, clearing a single task instance (UI, REST API or 
``airflowctl``) deletes the
+stored job name by default, so the next attempt submits a new job. Set 
``keep_task_state`` when
+clearing to keep it, and the next attempt then reconnects to the job or 
returns its result. Clearing
+a whole Dag run, marking a task as failed or success, and ``airflow tasks 
clear`` /
+``airflow dags clear`` still keep the job name before Airflow 3.5. From 
Airflow 3.5, every way of
+clearing deletes it by default.
+
+Clearing a running task leaves its job running by default, so the next attempt 
starts a second one
+and both are billed. To avoid that, set ``keep_task_state`` when clearing to 
reconnect, or set
+``cancel_on_kill=True`` to stop the old job.
+
+See :doc:`apache-airflow:core-concepts/resumable-tasks` for the ways of 
clearing, and for the
+``[state_store] clear_on_success`` setting.
+
+To opt out and always start a new job on retry, set ``durable=False``:
+
+.. code-block:: python
+
+  training = SageMakerTrainingOperator(
+      task_id="train_model",
+      config=training_config,
+      durable=False,
+  )
+
+Durable execution applies to the synchronous path. When ``deferrable=True`` and
+``wait_for_completion=True`` are both set, ``durable`` has no effect.
+
+Durable execution requires Airflow 3.3 or newer, since it relies on the task 
state store. Below
+3.3, ``durable`` has no effect either way: setting it explicitly only emits a 
warning, and the
+operator always starts a new job on retry, exactly as before this feature 
existed.
+
 .. _howto/operator:SageMakerModelOperator:
 
 Create an Amazon SageMaker model
diff --git 
a/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py 
b/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
index d27678e4444..9a8b2cd7712 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py
@@ -21,7 +21,7 @@ import json
 import time
 import urllib
 from collections.abc import Callable, Sequence
-from typing import TYPE_CHECKING, Any, ClassVar, cast
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from botocore.exceptions import ClientError
 
@@ -1326,9 +1326,7 @@ class SageMakerTrainingOperator(ResumableJobMixin, 
SageMakerBaseOperator):
             return self._execute_deferrable(context)
 
         self.execute_resumable(context)
-        # On reconnect the mixin returns poll_until_complete's value and never 
calls
-        # get_job_result, so the result is read off the operator instead.
-        return cast("dict[str, dict]", self._result)
+        return self.get_job_result(self._training_job_name, context)
 
     def submit_job(self, context: Context) -> str:
         # Not in execute(): on a reconnecting retry the S3Operations uploads 
would raise on the
@@ -1368,8 +1366,8 @@ class SageMakerTrainingOperator(ResumableJobMixin, 
SageMakerBaseOperator):
             raise
 
     def is_job_active(self, status: str) -> bool:
-        # Narrower than non_terminal_states on purpose: a "Stopping" job never 
produces a model, and
-        # wait_for_training_job accepts the resulting "Stopped" as done when 
print_log is off.
+        # Narrower than non_terminal_states on purpose: wait_for_training_job 
accepts the "Stopped" a
+        # "Stopping" job ends in as done when print_log is off, so 
reconnecting would report success.
         return status == "InProgress"
 
     def is_job_succeeded(self, status: str) -> bool:
diff --git 
a/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py 
b/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
index 3c5c9319372..cae0dd02b98 100644
--- 
a/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
+++ 
b/providers/amazon/tests/unit/amazon/aws/operators/test_sagemaker_training.py
@@ -185,14 +185,19 @@ class TestSageMakerTrainingOperator:
     @mock.patch.object(SageMakerHook, "create_training_job")
     @mock.patch.object(sagemaker, "serialize", return_value="")
     @mock.patch.object(SageMakerHook, "configure_s3_resources")
+    @mock.patch.object(SageMakerHook, "stop_training_job")
     def test_durable_reconnects_to_a_running_job(
-        self, mock_configure_s3, _, mock_training, mock_desc, mock_poll
+        self, mock_stop, mock_configure_s3, _, mock_training, mock_desc, 
mock_poll
     ):
         mock_desc.return_value = {"TrainingJobStatus": "InProgress"}
+        self.sagemaker.cancel_on_kill = True
         store = self._task_state_store("previously_running_job")
+        mock_poll.side_effect = lambda *a, **k: self.sagemaker.on_kill()
 
         result = self.sagemaker.execute({"task_state_store": store})
 
+        mock_stop.assert_called_once_with("previously_running_job")
+
         mock_training.assert_not_called()
         mock_configure_s3.assert_not_called()
         # autospec leaves the hook instance as args[0], so the job name is 
args[1].
@@ -200,6 +205,24 @@ class TestSageMakerTrainingOperator:
         assert result == {"Training": ""}
         assert self.sagemaker.serialized_training_data == ""
 
+    @pytest.mark.skipif(
+        not AIRFLOW_V_3_3_PLUS,
+        reason="ResumableJobMixin reconnect requires task_state_store, 
available in Airflow 3.3+",
+    )
+    @mock.patch.object(SageMakerHook, "wait_for_training_job")
+    @mock.patch.object(SageMakerHook, "describe_training_job")
+    def test_execute_returns_overridden_serialize_result(self, mock_desc, _):
+        class TrimmedTrainingOperator(SageMakerTrainingOperator):
+            def serialize_result(self, job_name: str) -> dict[str, dict]:
+                return {"trimmed": {"job_name": job_name}}
+
+        mock_desc.return_value = {"TrainingJobStatus": "InProgress"}
+        op = TrimmedTrainingOperator(task_id="test_sagemaker_operator", 
config=CREATE_TRAINING_PARAMS.copy())
+
+        result = op.execute({"task_state_store": 
self._task_state_store("previously_running_job")})
+
+        assert result == {"trimmed": {"job_name": "previously_running_job"}}
+
     @pytest.mark.skipif(
         not AIRFLOW_V_3_3_PLUS,
         reason="ResumableJobMixin reconnect requires task_state_store, 
available in Airflow 3.3+",

Reply via email to