amoghrajesh commented on code in PR #77161:
URL: https://github.com/apache/airflow/pull/77161#discussion_r4595710946
##########
task-sdk/src/airflow/sdk/bases/resumablejobmixin.py:
##########
@@ -122,88 +130,100 @@ def execute_resumable(self, context: Context) -> Any:
self.poll_until_complete(external_id, context)
return self.get_job_result(external_id, context)
+ decision = self.get_resume_decision(context)
+ if decision.action == "reconnect":
+ return self.poll_until_complete(decision.external_id, context)
+ if decision.action == "already_succeeded":
+ return self.get_job_result(decision.external_id, context)
+
+ external_id = self.submit_job(context)
+ self.persist_job_id(context, external_id)
+ self.poll_until_complete(external_id, context)
+ return self.get_job_result(external_id, context)
+
+ def get_resume_decision(self, context: Context) -> ResumeDecision:
Review Comment:
Handled in 3d371234d97
##########
task-sdk/src/airflow/sdk/bases/resumablejobmixin.py:
##########
@@ -122,88 +130,100 @@ def execute_resumable(self, context: Context) -> Any:
self.poll_until_complete(external_id, context)
return self.get_job_result(external_id, context)
+ decision = self.get_resume_decision(context)
+ if decision.action == "reconnect":
+ return self.poll_until_complete(decision.external_id, context)
+ if decision.action == "already_succeeded":
+ return self.get_job_result(decision.external_id, context)
+
+ external_id = self.submit_job(context)
+ self.persist_job_id(context, external_id)
+ self.poll_until_complete(external_id, context)
+ return self.get_job_result(external_id, context)
+
+ def get_resume_decision(self, context: Context) -> ResumeDecision:
+ """
+ Decide whether to reconnect to the stored job, take its result, or
submit a new one.
+
+ Never submits or polls, so a deferrable operator can call it before
``defer()``.
+ """
+ if not self.durable:
+ return ResumeDecision("submit")
+
ti = context.get("ti")
stats_tags = {**(ti.stats_tags if ti is not None else {}), "operator":
type(self).__name__}
- reconnect_to: Any = None
- already_succeeded_id: Any = None
-
with tracer.start_as_current_span("resumable_job.resume_decision") as
span:
span.set_attribute("operator", type(self).__name__)
span.set_attribute("resumable.external_id_key",
self.external_id_key)
task_state_store = context.get("task_state_store")
-
if task_state_store is None:
span.set_attribute("resumable.decision", "no_task_state_store")
self.log.warning(
"task_state_store not available in context, crash recovery
is disabled for this run"
)
- else:
- external_id = task_state_store.get(self.external_id_key)
- if external_id:
- stats.incr("resumable_job.reconnect_attempt",
tags=stats_tags)
-
- status = self.get_job_status(external_id, context)
-
- span.set_attribute("resumable.external_id",
str(external_id))
- span.set_attribute("resumable.prior_status", status)
-
- if self.is_job_active(status):
- # Job is still running, skip submission and reconnect
to it.
- span.set_attribute("resumable.decision", "reconnect")
- stats.incr("resumable_job.reconnect_success",
tags=stats_tags)
- self.log.info(
- "Reconnecting to existing job",
- external_id_key=self.external_id_key,
- external_id=external_id,
- status=status,
- )
- reconnect_to = external_id
- elif self.is_job_succeeded(status):
- # Job already finished successfully, skip polling and
return result directly.
- span.set_attribute("resumable.decision",
"already_succeeded")
- stats.incr("resumable_job.already_succeeded",
tags=stats_tags)
- self.log.info(
- "Job already completed successfully, skipping
resubmission",
- external_id_key=self.external_id_key,
- external_id=external_id,
- )
- already_succeeded_id = external_id
- else:
- # Job is in a terminal failed state, fall through and
submit a new job.
- span.set_attribute("resumable.decision",
"terminal_resubmit")
- stats.incr("resumable_job.terminal_resubmit",
tags=stats_tags)
- self.log.warning(
- "Prior job in terminal state, resubmitting fresh",
- external_id_key=self.external_id_key,
- external_id=external_id,
- status=status,
- )
- else:
- span.set_attribute("resumable.decision", "fresh_submit")
- stats.incr("resumable_job.fresh_submit", tags=stats_tags)
- self.log.debug(
- "No stored external ID found; submitting fresh job",
- external_id_key=self.external_id_key,
- )
-
- if reconnect_to is not None:
- return self.poll_until_complete(reconnect_to, context)
- if already_succeeded_id is not None:
- return self.get_job_result(already_succeeded_id, context)
- external_id = self.submit_job(context)
+ return ResumeDecision("submit")
+
+ external_id = task_state_store.get(self.external_id_key)
+ if not external_id:
Review Comment:
Handled in 3d371234d97
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]