amoghrajesh commented on code in PR #76918:
URL: https://github.com/apache/airflow/pull/76918#discussion_r4585618791


##########
providers/dbt/cloud/tests/unit/dbt/cloud/operators/test_dbt.py:
##########
@@ -1058,6 +1063,168 @@ def test_run_job_operator_link(
         )
 
 
+STORED_RUN_ID = 9999
+STORED_RUN_URL = EXPECTED_JOB_RUN_OP_EXTRA_LINK.format(
+    account_id=DEFAULT_ACCOUNT_ID, project_id=PROJECT_ID, run_id=STORED_RUN_ID
+)
+STORED_RUN_API_URL = 
f"https://cloud.getdbt.com/api/v2/accounts/{DEFAULT_ACCOUNT_ID}/runs/{STORED_RUN_ID}/";
+
+
[email protected](
+    not AIRFLOW_V_3_3_PLUS, reason="task_state_store (durable execution) 
requires Airflow 3.3+"
+)
[email protected](DbtCloudHook, "wait_for_job_run_status", autospec=True, 
return_value=True)
[email protected](
+    DbtCloudHook,
+    "trigger_job_run",
+    autospec=True,
+    return_value=mock_response_json(DEFAULT_ACCOUNT_JOB_RUN_RESPONSE),
+)
+class TestDbtCloudRunJobOperatorDurable:
+    def setup_method(self):
+        self.dag = DAG("test_dbt_cloud_job_run_op_durable", schedule=None, 
start_date=DEFAULT_DATE)
+        self.ti = MagicMock(stats_tags={}, try_number=1)
+        self.task_store = MagicMock(spec_set=["get", "set"])
+        self.task_store.get.return_value = None
+        self.context = {"ti": self.ti, "task_state_store": self.task_store}
+
+    def _operator(self, **kwargs):
+        return DbtCloudRunJobOperator(
+            task_id=TASK_ID, dbt_cloud_conn_id=ACCOUNT_ID_CONN, job_id=JOB_ID, 
dag=self.dag, **kwargs
+        )
+
+    def _assert_run_xcoms(self, run_id, run_url):
+        assert self.ti.xcom_push.call_args_list == [
+            call(key="job_run_url", value=run_url),
+            call(key="job_run_id", value=run_id),
+        ]
+
+    def test_fresh_submit_persists_run_id(self, mock_trigger, mock_wait):
+        assert self._operator().execute(self.context) == RUN_ID
+
+        mock_trigger.assert_called_once()
+        mock_wait.assert_called_once()
+        self.task_store.set.assert_called_once_with("dbt_cloud_job_run_id", 
RUN_ID)
+        self._assert_run_xcoms(RUN_ID, 
DEFAULT_ACCOUNT_JOB_RUN_RESPONSE["data"]["href"])
+
+    @pytest.mark.parametrize(
+        ("status", "waits"),
+        [
+            (DbtCloudJobRunStatus.QUEUED, True),
+            (DbtCloudJobRunStatus.STARTING, True),
+            (DbtCloudJobRunStatus.RUNNING, True),
+            (DbtCloudJobRunStatus.SUCCESS, False),
+        ],
+    )
+    def test_stored_run_is_reused(self, mock_trigger, mock_wait, 
requests_mock, status, waits):
+        self.task_store.get.return_value = STORED_RUN_ID
+        requests_mock.get(
+            STORED_RUN_API_URL,
+            json={"data": {"id": STORED_RUN_ID, "status": status.value, 
"href": STORED_RUN_URL}},
+        )
+        operator = self._operator()
+
+        assert operator.execute(self.context) == STORED_RUN_ID
+
+        mock_trigger.assert_not_called()
+        assert mock_wait.called is waits
+        self.task_store.set.assert_not_called()
+        assert operator.run_id == STORED_RUN_ID
+        self._assert_run_xcoms(STORED_RUN_ID, STORED_RUN_URL)
+
+    @pytest.mark.parametrize(
+        "response",
+        [
+            {
+                "json": {
+                    "data": {
+                        "id": STORED_RUN_ID,
+                        "status": DbtCloudJobRunStatus.ERROR.value,
+                        "href": STORED_RUN_URL,
+                    }
+                }
+            },
+            {
+                "json": {
+                    "data": {
+                        "id": STORED_RUN_ID,
+                        "status": DbtCloudJobRunStatus.CANCELLED.value,
+                        "href": STORED_RUN_URL,
+                    }
+                }
+            },
+            {"status_code": 404, "reason": "Not Found"},
+        ],
+        ids=["error", "cancelled", "not_found"],
+    )
+    def test_stored_run_not_reusable_submits_fresh(self, mock_trigger, 
mock_wait, requests_mock, response):

Review Comment:
   Added `test_stored_errored_run_retries_from_failure`: stored ERROR run, 
`retry_from_failure=True`, try 2, and it asserts the trigger call gets 
`retry_from_failure=True`.



##########
providers/dbt/cloud/src/airflow/providers/dbt/cloud/operators/dbt.py:
##########
@@ -243,68 +274,103 @@ def execute(self, context: Context):
 
         self.job_id = self._resolve_job_id()
 
-        self.run_id, job_run_url = self._get_or_trigger_run(context)
+        if self.wait_for_termination:
+            self.execute_resumable(context)
+            return self.run_id
 
+        if self.deferrable is True:
+            warnings.warn(
+                "Argument `wait_for_termination` is False and `deferrable` is 
True , hence "
+                "`deferrable` parameter doesn't have any effect",
+                UserWarning,
+                stacklevel=2,
+            )
+        self.run_id, self._job_run_url = self._get_or_trigger_run(context)
         # Push the ``job_run_url`` and ``job_run_id`` value to XCom regardless 
of what happens during execution.
         # This enables job monitoring via the operator link and provides 
direct access
         # to the job run ID without requiring users to parse the URL manually
-        context["ti"].xcom_push(key="job_run_url", value=job_run_url)
+        self._push_run_xcoms(context)
+        return self.run_id
+
+    def _defer_until_complete(self, run_id: int) -> None:
+        # Derive absolute deadlines for deferrable execution.
+        # execution_timeout is a hard task-level limit (cancels the job),
+        # while timeout only limits how long we wait for the job to finish.
+        # If both are set, the earliest deadline wins.
+        end_time = time.time() + self.timeout
+        execution_deadline = None
+        if self.execution_timeout is not None:
+            execution_deadline = time.time() + 
self.execution_timeout.total_seconds()
+
+        job_run_info = JobRunInfo(account_id=self.account_id, run_id=run_id)
+        job_run_status = self.hook.get_job_run_status(**job_run_info)
+        if DbtCloudJobRunStatus.is_terminal(job_run_status):
+            self._handle_terminal_status(job_run_status)
+            return
+        self.defer(
+            timeout=None,
+            trigger=DbtCloudRunJobTrigger(
+                conn_id=self.dbt_cloud_conn_id,
+                run_id=run_id,
+                end_time=end_time,
+                execution_deadline=execution_deadline,
+                account_id=self.account_id,
+                poll_interval=self.check_interval,
+                hook_params=self.hook_params,
+            ),
+            method_name="execute_complete",
+        )
+
+    def _push_run_xcoms(self, context: Context) -> None:
+        context["ti"].xcom_push(key="job_run_url", value=self._job_run_url)
         context["ti"].xcom_push(key="job_run_id", value=self.run_id)
+        self._run_xcoms_pushed = True
 
-        if self.wait_for_termination and isinstance(self.run_id, int):
-            if self.deferrable is False:
-                self.log.info("Waiting for job run %s to terminate.", 
self.run_id)
-
-                if self.hook.wait_for_job_run_status(
-                    run_id=self.run_id,
-                    account_id=self.account_id,
-                    expected_statuses=DbtCloudJobRunStatus.SUCCESS.value,
-                    check_interval=self.check_interval,
-                    timeout=self.timeout,
-                ):
-                    self.log.info("Job run %s has completed successfully.", 
self.run_id)
-                else:
-                    raise DbtCloudJobRunException(f"Job run {self.run_id} has 
failed or has been cancelled.")
-
-                return self.run_id
-
-            # Derive absolute deadlines for deferrable execution.
-            # execution_timeout is a hard task-level limit (cancels the job),
-            # while timeout only limits how long we wait for the job to finish.
-            # If both are set, the earliest deadline wins.
-            end_time = time.time() + self.timeout
-            execution_deadline = None
-            if self.execution_timeout is not None:
-                execution_deadline = time.time() + 
self.execution_timeout.total_seconds()
-
-            job_run_info = JobRunInfo(account_id=self.account_id, 
run_id=self.run_id)
-            job_run_status = self.hook.get_job_run_status(**job_run_info)
-            if not DbtCloudJobRunStatus.is_terminal(job_run_status):
-                self.defer(
-                    timeout=None,
-                    trigger=DbtCloudRunJobTrigger(
-                        conn_id=self.dbt_cloud_conn_id,
-                        run_id=self.run_id,
-                        end_time=end_time,
-                        execution_deadline=execution_deadline,
-                        account_id=self.account_id,
-                        poll_interval=self.check_interval,
-                        hook_params=self.hook_params,
-                    ),
-                    method_name="execute_complete",
-                )
-            else:
-                return self._handle_terminal_status(job_run_status)
+    def submit_job(self, context: Context) -> int:
+        self.run_id, self._job_run_url = self._get_or_trigger_run(context)

Review Comment:
   The stored run wins. It is checked before `submit_job`, and 
`reuse_existing_run` only applies inside `submit_job`. Added 
`test_stored_run_takes_precedence_over_reuse_existing_run` to pin that.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to