shahar1 commented on code in PR #75658:
URL: https://github.com/apache/airflow/pull/75658#discussion_r4572309840


##########
providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/kueue.py:
##########
@@ -133,3 +164,119 @@ def __init__(self, queue_name: str, *args, **kwargs) -> 
None:
             self.suspend = True
         self.labels.update({"kueue.x-k8s.io/queue-name": self.queue_name})
         self.annotations.update({"kueue.x-k8s.io/queue-name": self.queue_name})
+
+    def execute(self, context: Context):
+        if not self.wait_until_job_complete:
+            return super().execute(context)
+
+        if self.parallelism is not None and self.parallelism < 1:
+            raise ValueError("parallelism cannot be less than 1 with 
`wait_until_job_complete=True`.")
+        self.name = self._set_name(self.name)
+        self.job_request_obj = self.build_job_request_obj(context)
+        self.job = self.create_job(job_request_obj=self.job_request_obj)  # 
must set `self.job` for `on_kill`
+
+        ti = context["ti"]
+        ti.xcom_push(key="job_name", value=self.job.metadata.name)
+        ti.xcom_push(key="job_namespace", value=self.job.metadata.namespace)
+
+        if self.deferrable:
+            self.defer(
+                trigger=KubernetesKueueJobTrigger(
+                    job_name=self.job.metadata.name,
+                    job_namespace=self.job.metadata.namespace,
+                    base_container_name=self.base_container_name,
+                    kubernetes_conn_id=self.kubernetes_conn_id,
+                    cluster_context=self.cluster_context,
+                    config_file=self.config_file,
+                    in_cluster=self.in_cluster,
+                    poll_interval=self.job_poll_interval,
+                    do_xcom_push=self.do_xcom_push,
+                ),
+                method_name="execute_complete",
+                # Airflow 3 does not apply execution_timeout to the time spent 
deferred
+                timeout=self.execution_timeout,
+            )
+
+        return self.execute_complete(
+            context, self._wait_until_job_finished(self.job.metadata.name, 
self.job.metadata.namespace)
+        )
+
+    def _wait_until_job_finished(self, job_name: str, namespace: str) -> 
dict[str, Any]:
+        """Poll the Job until it is finished, and return the event 
``KubernetesKueueJobTrigger`` yields."""
+        state: KueueJobState | None = None
+        xcom_results: dict[str, str] = {}
+        while True:
+            job = self.hook.get_job_status(job_name=job_name, 
namespace=namespace)
+            if (new_state := get_job_state(job)) != state:
+                state = new_state
+                self.log.info("The job '%s' %s", job_name, 
JOB_STATE_MESSAGES[state])
+            if state == KueueJobState.FINISHED:
+                return build_job_event(job, xcom_results)
+            if state == KueueJobState.RUNNING and self.do_xcom_push:
+                collect_xcom_results(
+                    self.client, self.pod_manager, job_name, namespace, 
self.base_container_name, xcom_results
+                )
+            time.sleep(self.job_poll_interval)
+
+    def execute_complete(self, context: Context, event: dict, **kwargs):
+        # The pods found before a preemption by Kueue were deleted: read the 
ones the Job has now
+        pods = sorted(
+            list_job_pods(self.client, event["name"], event["namespace"]), 
key=lambda pod: pod.metadata.name
+        )
+        for pod in pods:
+            self.log_matching_pod(pod=pod, context=context)
+        if self.get_logs and event["status"] == "error":
+            # super() raises for a failed Job before it writes the logs of the 
pods
+            for pod in pods:
+                self._write_logs(pod)
+        xcom_result = get_succeeded_pod_xcoms(pods, event["xcom_result"])
+        result = super().execute_complete(
+            context,
+            {
+                **event,
+                "pod_names": [pod.metadata.name for pod in pods],
+                "pod_namespace": event["namespace"],
+                "xcom_result": xcom_result,
+            },
+            **kwargs,
+        )
+        # Only reached for a successful Job: super() raises for a failed one
+        succeeded = (event["job"].get("status") or {}).get("succeeded") or 0
+        if self.do_xcom_push and len(xcom_result) < succeeded:
+            raise XComRetrievalError(
+                f"Read the XCom of {len(xcom_result)} of the {succeeded} pods 
that succeeded in the job "
+                f"'{event['name']}': the other pods were deleted, or their 
XCom could not be read"
+            )
+        return result
+
+    def resume_execution(self, next_method: str, next_kwargs: dict[str, Any] | 
None, context: Context):
+        if next_method == "__fail__" and (next_kwargs or {}).get("error") in 
_DEFERRAL_TIMEOUT_ERRORS:
+            self._kill_job_of_timed_out_deferral(context)
+        return super().resume_execution(next_method, next_kwargs, context)
+
+    def _kill_job_of_timed_out_deferral(self, context: Context) -> None:
+        """
+        Delete the Job when the deferral timed out, as ``on_kill`` does when 
the sync wait times out.
+
+        ``on_kill`` is not called for a deferred task, and a Job that Kueue 
never admits would otherwise
+        stay in its queue, and be run if Kueue admits it later.
+        """
+        ti = context["ti"]
+        name = ti.xcom_pull(task_ids=ti.task_id, key="job_name")
+        namespace = ti.xcom_pull(task_ids=ti.task_id, key="job_namespace")

Review Comment:
   **major**: the timeout cleanup deletes no Job when the task is mapped. The 
new docs promise: "When the timeout is reached, the task fails and the Job is 
deleted, so that Kueue does not run it later."
   
   Without `map_indexes`, `xcom_pull(task_ids=ti.task_id, ...)` returns the 
value of every map index of a mapped task, not only this one:
   
   - On Airflow 3, `RuntimeTaskInstance.xcom_pull` takes its "map_indexes was 
not specified — fetch all map indexes" branch 
(`task-sdk/.../execution_time/task_runner.py`). It returns a list as soon as 
more than one map index pushed `job_name`. Its docstring says otherwise, but 
the code does this.
   - On Airflow 2.11, `TaskInstance.xcom_pull` returns a 
`LazyXComSelectSequence` for a mapped task.
   
   `name` and `namespace` are then collections, so `delete_namespaced_job` gets 
no valid Job name. The Job stays queued in Kueue and runs once Kueue admits it, 
which is the case this method exists to prevent. Fanning a Kueue job out with 
`.expand()` is a normal use, so this is not a corner case.
   
   ```suggestion
           name = ti.xcom_pull(task_ids=ti.task_id, key="job_name", 
map_indexes=ti.map_index)
           namespace = ti.xcom_pull(task_ids=ti.task_id, key="job_namespace", 
map_indexes=ti.map_index)
   ```
   
   The `xcom_pull` mock in 
`test_resume_execution_deletes_job_when_deferral_timed_out` takes only 
`(task_ids, key)`, so it will need `map_indexes` too. Giving the TI a 
`map_index` of `1` there would cover the mapped case.
   
   *Flagged by both the primary and adversarial reviewers.*
   



##########
providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/triggers/kueue.py:
##########
@@ -0,0 +1,276 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Wait for a Kubernetes Job run through Kueue."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections.abc import AsyncIterator, Iterable
+from enum import Enum
+from functools import cached_property
+from typing import TYPE_CHECKING, Any
+
+from kubernetes.client.rest import ApiException
+
+from airflow.providers.cncf.kubernetes.hooks.kubernetes import (
+    JOB_FINAL_STATUS_CONDITION_TYPES,
+    AsyncKubernetesHook,
+    KubernetesHook,
+)
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import 
generic_api_retry
+from airflow.providers.cncf.kubernetes.utils.container import 
container_is_running, container_is_terminated
+from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager, 
XComRetrievalError
+from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
+from airflow.triggers.base import BaseTrigger, TriggerEvent
+
+if TYPE_CHECKING:
+    from kubernetes.client import CoreV1Api, models as k8s
+
+# Set by the Job controller on the pods it creates for a Job
+JOB_NAME_LABEL = "batch.kubernetes.io/job-name"
+
+
+class KueueJobState(str, Enum):
+    """State of a Job run through Kueue."""
+
+    SUSPENDED = "suspended"
+    RUNNING = "running"
+    FINISHED = "finished"
+
+
+JOB_STATE_MESSAGES = {
+    KueueJobState.SUSPENDED: "is suspended: waiting for admission by Kueue, or 
preempted by Kueue",
+    KueueJobState.RUNNING: "is admitted by Kueue: its pods are running",
+    KueueJobState.FINISHED: "is finished",
+}
+
+
+def get_job_state(job: k8s.V1Job) -> KueueJobState:
+    """Return the state of a Job run through Kueue; finished means Complete or 
Failed."""
+    conditions = (job.status.conditions if job.status else None) or []
+    if any(c.type in JOB_FINAL_STATUS_CONDITION_TYPES and c.status == "True" 
for c in conditions):
+        return KueueJobState.FINISHED
+    # The Job is created suspended; Kueue resumes it on admission and suspends 
it again on preemption
+    if job.spec and job.spec.suspend:
+        return KueueJobState.SUSPENDED
+    return KueueJobState.RUNNING
+
+
+def build_job_event(job: k8s.V1Job, xcom_results: dict[str, str]) -> dict[str, 
Any]:
+    """
+    Build the event of a finished Job, as passed to 
``KubernetesStartKueueJobOperator.execute_complete``.
+
+    :param job: the finished Job.
+    :param xcom_results: the raw XCom read from each pod, by pod name.
+    """
+    error_message = KubernetesHook.is_job_failed(job=job)
+    return {
+        "name": job.metadata.name,
+        "namespace": job.metadata.namespace,
+        "status": "error" if error_message else "success",
+        "message": f"Job failed with error: {error_message}"
+        if error_message
+        else "Job completed successfully",
+        "job": job.to_dict(),
+        "xcom_result": xcom_results,
+    }
+
+
+def get_succeeded_pod_xcoms(pods: Iterable[k8s.V1Pod], xcom_results: dict[str, 
str]) -> list[str]:
+    """Return the XCom read from the pods that succeeded, skipping failed pods 
and preempted runs."""
+    return [
+        xcom_results[pod.metadata.name]
+        for pod in pods
+        if pod.metadata.name in xcom_results and pod.status and 
pod.status.phase == "Succeeded"
+    ]
+
+
+@generic_api_retry
+def list_job_pods(client: CoreV1Api, job_name: str, namespace: str) -> 
list[k8s.V1Pod]:
+    """Return the pods the Job has now."""
+    return client.list_namespaced_pod(
+        namespace=namespace, label_selector=f"{JOB_NAME_LABEL}={job_name}"

Review Comment:
   **minor**: with `manual_selector=True`, this selector matches no pod. The 
Kubernetes API reference for `manualSelector` says: "When false or unset, the 
system pick labels unique to this job and appends those labels to the pod 
template.  When true, the user is responsible for picking unique labels and 
specifying the selector."
   
   So `batch.kubernetes.io/job-name` is added to the pods only when 
`manualSelector` is unset. `KubernetesJobOperator` accepts `manual_selector` 
and `selector` and passes them to the Job. Its own `get_pods` keeps working 
with them because it selects on the Airflow TI labels of the pod template 
(`_build_find_pod_label_selector`).
   
   Here, with `do_xcom_push=True`, `collect_xcom_results` never finds a pod, so 
it never stops the sidecars. The Job never finishes, and the task waits until 
`execution_timeout`, or forever without one. Without XCom, logs and the 
`on_finish_action` cleanup are skipped without a word.
   
   Selecting on the Job's own `spec.selector.match_labels` would work in both 
modes. When Kubernetes generates the selector, it holds 
`batch.kubernetes.io/controller-uid`, which is also unique across Jobs that 
reuse a name. Selecting on the same labels as `get_pods` would work too.
   
   *Flagged by the adversarial reviewer; cross-checked.*
   



##########
providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/kueue.py:
##########
@@ -133,3 +164,119 @@ def __init__(self, queue_name: str, *args, **kwargs) -> 
None:
             self.suspend = True
         self.labels.update({"kueue.x-k8s.io/queue-name": self.queue_name})
         self.annotations.update({"kueue.x-k8s.io/queue-name": self.queue_name})
+
+    def execute(self, context: Context):
+        if not self.wait_until_job_complete:
+            return super().execute(context)
+
+        if self.parallelism is not None and self.parallelism < 1:
+            raise ValueError("parallelism cannot be less than 1 with 
`wait_until_job_complete=True`.")
+        self.name = self._set_name(self.name)
+        self.job_request_obj = self.build_job_request_obj(context)
+        self.job = self.create_job(job_request_obj=self.job_request_obj)  # 
must set `self.job` for `on_kill`
+
+        ti = context["ti"]
+        ti.xcom_push(key="job_name", value=self.job.metadata.name)
+        ti.xcom_push(key="job_namespace", value=self.job.metadata.namespace)
+
+        if self.deferrable:
+            self.defer(
+                trigger=KubernetesKueueJobTrigger(
+                    job_name=self.job.metadata.name,
+                    job_namespace=self.job.metadata.namespace,
+                    base_container_name=self.base_container_name,
+                    kubernetes_conn_id=self.kubernetes_conn_id,
+                    cluster_context=self.cluster_context,
+                    config_file=self.config_file,
+                    in_cluster=self.in_cluster,
+                    poll_interval=self.job_poll_interval,
+                    do_xcom_push=self.do_xcom_push,
+                ),
+                method_name="execute_complete",
+                # Airflow 3 does not apply execution_timeout to the time spent 
deferred
+                timeout=self.execution_timeout,
+            )
+
+        return self.execute_complete(
+            context, self._wait_until_job_finished(self.job.metadata.name, 
self.job.metadata.namespace)
+        )
+
+    def _wait_until_job_finished(self, job_name: str, namespace: str) -> 
dict[str, Any]:
+        """Poll the Job until it is finished, and return the event 
``KubernetesKueueJobTrigger`` yields."""
+        state: KueueJobState | None = None
+        xcom_results: dict[str, str] = {}
+        while True:
+            job = self.hook.get_job_status(job_name=job_name, 
namespace=namespace)
+            if (new_state := get_job_state(job)) != state:
+                state = new_state
+                self.log.info("The job '%s' %s", job_name, 
JOB_STATE_MESSAGES[state])
+            if state == KueueJobState.FINISHED:
+                return build_job_event(job, xcom_results)
+            if state == KueueJobState.RUNNING and self.do_xcom_push:
+                collect_xcom_results(
+                    self.client, self.pod_manager, job_name, namespace, 
self.base_container_name, xcom_results
+                )
+            time.sleep(self.job_poll_interval)
+
+    def execute_complete(self, context: Context, event: dict, **kwargs):
+        # The pods found before a preemption by Kueue were deleted: read the 
ones the Job has now
+        pods = sorted(
+            list_job_pods(self.client, event["name"], event["namespace"]), 
key=lambda pod: pod.metadata.name
+        )
+        for pod in pods:
+            self.log_matching_pod(pod=pod, context=context)
+        if self.get_logs and event["status"] == "error":
+            # super() raises for a failed Job before it writes the logs of the 
pods
+            for pod in pods:
+                self._write_logs(pod)
+        xcom_result = get_succeeded_pod_xcoms(pods, event["xcom_result"])

Review Comment:
   **minor**: a task deferred before the provider upgrade fails when it resumes 
here. Before this PR, the deferrable path of this operator was 
`KubernetesJobOperator.execute_deferrable`, which defers to 
`KubernetesJobTrigger` with `method_name="execute_complete"`. The event of that 
trigger carries `xcom_result` as a list, or `None` without XCom 
(`triggers/job.py`).
   
   `get_succeeded_pod_xcoms` expects a dict keyed by pod name:
   
   - `None` raises `TypeError` on `pod.metadata.name in xcom_results` as soon 
as the Job has a pod.
   - A list matches no pod name, so the count check after `super()` raises 
`XComRetrievalError`.
   
   Either way, a Job that completed fails the task, and the retry runs it 
again. Kueue Jobs can stay deferred for hours, so some will be in flight during 
an upgrade.
   
   Handing an event without a dict `xcom_result` to `super()` would keep those 
tasks working, for example at the top of this method:
   
   ```python
   if not isinstance(event.get("xcom_result"), dict):
       # Event of a KubernetesJobTrigger deferred by an earlier provider version
       return super().execute_complete(context, event, **kwargs)
   ```
   
   *Flagged by the adversarial reviewer; cross-checked.*
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to