ferruzzi commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4633324255


##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -456,6 +510,184 @@ def __init__(
     def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
         stats.incr("scheduler_heartbeat", 1, 1)
 
+    def _check_executor_health(self) -> None:
+        """
+        Kick off a health check for each reporting executor, in the background.
+
+        Called on its own timer (``[scheduler] 
executor_health_check_interval``), independent of the
+        scheduling loop and of each executor's own 
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+        A new check for a given executor is skipped while its previous check 
is still running, so a
+        slow or hung dependency can only ever delay that executor's own health 
being updated, never
+        pile up work or block this (or any other) scheduler tick.
+
+        ``get_health()`` is passed a timeout, but an implementation that 
ignores it can still leave its
+        future pending forever. Once a pending future is a few intervals old, 
it's reported as stalled:
+        the scheduler records UNHEALTHY itself, since a check that hasn't 
answered isn't still-healthy,
+        and logs and emits a metric so the stall is visible. If the stuck call 
does eventually return,
+        its result overwrites this one.
+        """
+        for executor in self.executors:
+            key = _executor_health_key(executor)
+            if key is None or key not in self._executor_health_keys:
+                # No name, or doesn't override get_health(): there's nothing 
to check or record.
+                continue
+            future, started = self._executor_health_check_futures.get(key, 
(None, 0.0))
+            if future is not None and not future.done():
+                stall_after = (
+                    self._executor_health_check_interval * 
self._executor_health_check_stall_after_intervals
+                )
+                if time.monotonic() - started > stall_after and key not in 
self._stalled_health_checks:
+                    self._stalled_health_checks.add(key)
+                    self.log.warning("Health check for executor %s has stopped 
responding", key)
+                    stats.incr(
+                        "scheduler.executor_health_check.stalled",
+                        tags=prune_dict(
+                            {"executor": type(executor).__name__, "team_name": 
executor.team_name}
+                        ),
+                    )
+                    try:
+                        self._record_executor_health_result(
+                            key, executor.team_name, 
HealthCheckResult.UNHEALTHY
+                        )
+                    except Exception:
+                        self.log.exception("Failed to record health check 
result for executor %s", key)
+                else:
+                    self.log.debug(
+                        "Skipping health check for executor %s, the previous 
check hasn't finished", key
+                    )
+                continue
+            self._stalled_health_checks.discard(key)
+            future = Future()
+            threading.Thread(
+                target=self._run_executor_health_check_and_resolve,
+                args=(executor, key, future),
+                name=f"executor-health-check-{key}",
+                daemon=True,
+            ).start()
+            self._executor_health_check_futures[key] = (future, 
time.monotonic())
+
+    def _run_executor_health_check_and_resolve(
+        self, executor: BaseExecutor, key: str, future: Future
+    ) -> None:
+        """Run one executor's health check and resolve its tracking future, 
even if it raises."""
+        try:
+            self._run_executor_health_check(executor, key)
+        finally:
+            future.set_result(None)
+
+    def _run_executor_health_check(self, executor: BaseExecutor, key: str) -> 
None:
+        """Run one executor's get_health() off the scheduler thread and record 
an explicit result."""
+        try:
+            healthy = 
executor.get_health(timeout=self._executor_health_check_timeout)
+        except NotImplementedError:
+            healthy = None
+        except Exception:
+            self.log.exception("Health check for executor %s raised an 
exception", key)
+            healthy = False
+        if healthy is None:
+            result = HealthCheckResult.UNKNOWN
+        elif healthy:
+            result = HealthCheckResult.HEALTHY
+        else:
+            self.log.warning("Executor %s reported itself unhealthy", key)
+            result = HealthCheckResult.UNHEALTHY
+        try:
+            self._record_executor_health_result(key, executor.team_name, 
result)
+        except Exception:
+            self.log.exception("Failed to record health check result for 
executor %s", key)
+
+    @provide_session
+    def _touch_executor_heartbeats(self, *, session: Session = NEW_SESSION) -> 
None:
+        """
+        Keep every reporting executor's Job row heartbeat moving on the 
scheduler's own cadence.
+
+        ``get_health()`` itself only runs every 
``executor_health_check_interval`` (default 30s), far
+        slower than a Job row can go quiet for before 
``health_check_threshold`` calls it dead.
+        Updating the heartbeat only when a check finishes would make a 
perfectly healthy executor
+        flicker to UNHEALTHY between checks, every cycle. Keeping the 
heartbeat on the scheduler's own
+        short cadence instead, the same way it already does for its own Job 
row, means the row's
+        liveness reflects "the scheduler is still running and still intends to 
check this executor",
+        while ``health_check_result`` -- not this heartbeat -- carries 
whatever the last actual check
+        found.
+        """
+        if not self._executor_health_keys:
+            return
+        now = timezone.utcnow()
+        for key, team_name in self._executor_health_keys.items():
+            job = self._get_or_create_executor_job(key, team_name, 
session=session)
+            job.latest_heartbeat = now
+        session.commit()

Review Comment:
   A method which uses the @provide_session decorator shouldn't have to do 
session.commit(), if this is required then something else is going on.  the 
decorator should be commiting the session when it exits.  Try removing it and 
see if it still works.



##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -456,6 +510,184 @@ def __init__(
     def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
         stats.incr("scheduler_heartbeat", 1, 1)
 
+    def _check_executor_health(self) -> None:
+        """
+        Kick off a health check for each reporting executor, in the background.
+
+        Called on its own timer (``[scheduler] 
executor_health_check_interval``), independent of the
+        scheduling loop and of each executor's own 
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+        A new check for a given executor is skipped while its previous check 
is still running, so a
+        slow or hung dependency can only ever delay that executor's own health 
being updated, never
+        pile up work or block this (or any other) scheduler tick.
+
+        ``get_health()`` is passed a timeout, but an implementation that 
ignores it can still leave its
+        future pending forever. Once a pending future is a few intervals old, 
it's reported as stalled:
+        the scheduler records UNHEALTHY itself, since a check that hasn't 
answered isn't still-healthy,
+        and logs and emits a metric so the stall is visible. If the stuck call 
does eventually return,
+        its result overwrites this one.
+        """
+        for executor in self.executors:
+            key = _executor_health_key(executor)
+            if key is None or key not in self._executor_health_keys:
+                # No name, or doesn't override get_health(): there's nothing 
to check or record.
+                continue
+            future, started = self._executor_health_check_futures.get(key, 
(None, 0.0))
+            if future is not None and not future.done():
+                stall_after = (
+                    self._executor_health_check_interval * 
self._executor_health_check_stall_after_intervals
+                )
+                if time.monotonic() - started > stall_after and key not in 
self._stalled_health_checks:
+                    self._stalled_health_checks.add(key)
+                    self.log.warning("Health check for executor %s has stopped 
responding", key)
+                    stats.incr(
+                        "scheduler.executor_health_check.stalled",
+                        tags=prune_dict(
+                            {"executor": type(executor).__name__, "team_name": 
executor.team_name}
+                        ),
+                    )
+                    try:
+                        self._record_executor_health_result(
+                            key, executor.team_name, 
HealthCheckResult.UNHEALTHY
+                        )
+                    except Exception:
+                        self.log.exception("Failed to record health check 
result for executor %s", key)
+                else:
+                    self.log.debug(
+                        "Skipping health check for executor %s, the previous 
check hasn't finished", key
+                    )
+                continue
+            self._stalled_health_checks.discard(key)
+            future = Future()
+            threading.Thread(
+                target=self._run_executor_health_check_and_resolve,
+                args=(executor, key, future),
+                name=f"executor-health-check-{key}",
+                daemon=True,
+            ).start()
+            self._executor_health_check_futures[key] = (future, 
time.monotonic())
+
+    def _run_executor_health_check_and_resolve(
+        self, executor: BaseExecutor, key: str, future: Future
+    ) -> None:
+        """Run one executor's health check and resolve its tracking future, 
even if it raises."""
+        try:
+            self._run_executor_health_check(executor, key)
+        finally:
+            future.set_result(None)
+
+    def _run_executor_health_check(self, executor: BaseExecutor, key: str) -> 
None:
+        """Run one executor's get_health() off the scheduler thread and record 
an explicit result."""
+        try:
+            healthy = 
executor.get_health(timeout=self._executor_health_check_timeout)
+        except NotImplementedError:
+            healthy = None
+        except Exception:
+            self.log.exception("Health check for executor %s raised an 
exception", key)
+            healthy = False
+        if healthy is None:
+            result = HealthCheckResult.UNKNOWN
+        elif healthy:
+            result = HealthCheckResult.HEALTHY
+        else:
+            self.log.warning("Executor %s reported itself unhealthy", key)
+            result = HealthCheckResult.UNHEALTHY
+        try:
+            self._record_executor_health_result(key, executor.team_name, 
result)
+        except Exception:
+            self.log.exception("Failed to record health check result for 
executor %s", key)
+
+    @provide_session
+    def _touch_executor_heartbeats(self, *, session: Session = NEW_SESSION) -> 
None:
+        """
+        Keep every reporting executor's Job row heartbeat moving on the 
scheduler's own cadence.
+
+        ``get_health()`` itself only runs every 
``executor_health_check_interval`` (default 30s), far
+        slower than a Job row can go quiet for before 
``health_check_threshold`` calls it dead.
+        Updating the heartbeat only when a check finishes would make a 
perfectly healthy executor
+        flicker to UNHEALTHY between checks, every cycle. Keeping the 
heartbeat on the scheduler's own
+        short cadence instead, the same way it already does for its own Job 
row, means the row's
+        liveness reflects "the scheduler is still running and still intends to 
check this executor",
+        while ``health_check_result`` -- not this heartbeat -- carries 
whatever the last actual check
+        found.
+        """
+        if not self._executor_health_keys:
+            return
+        now = timezone.utcnow()
+        for key, team_name in self._executor_health_keys.items():
+            job = self._get_or_create_executor_job(key, team_name, 
session=session)
+            job.latest_heartbeat = now
+        session.commit()
+
+    def _get_or_create_executor_job(
+        self, executor_key: str, team_name: str | None, *, session: Session
+    ) -> Job:
+        """
+        Return this scheduler's own Job row for ``executor_key``, creating it 
if needed.
+
+        Keyed by ``owner_job_id`` (this scheduler's own Job id), not just 
``hostname``: two schedulers
+        running the same executor against shared infrastructure (e.g. the same 
Celery broker) each get
+        their own row, so one scheduler's healthy report can never paper over 
another's failing one.
+        A restarted scheduler gets a fresh row for the same reason its own Job 
row is fresh -- a new
+        id -- instead of quietly taking over a row left behind by its previous 
instance.
+
+        Inserted directly rather than through ``Job(...)``: that constructor 
fires the ``on_starting``
+        listener hook, meant for "a component is starting", but the scheduler 
is already running and
+        nothing here ever fires the matching ``before_stopping``. A plain 
insert sidesteps that, the
+        same way the Execution API's job-registration endpoint does for Jobs 
it creates on another
+        component's behalf.
+        """
+        job = session.scalar(
+            select(Job).where(
+                Job.job_type == EXECUTOR_JOB_TYPE,
+                Job.executor_class == executor_key,
+                Job.owner_job_id == self.job.id,
+                Job.end_date.is_(None),
+            )
+        )
+        if job is None:
+            now = timezone.utcnow()
+            inserted = session.execute(
+                insert(Job).values(
+                    job_type=EXECUTOR_JOB_TYPE,
+                    executor_class=executor_key,
+                    owner_job_id=self.job.id,
+                    health_check_result=HealthCheckResult.UNKNOWN,
+                    state=JobState.RUNNING,
+                    start_date=now,
+                    latest_heartbeat=now,
+                    hostname=get_hostname(),
+                    unixname=getuser(),
+                )
+            )
+            job_id = cast("CursorResult", inserted).inserted_primary_key[0]  # 
type: ignore[index]
+            if team_name:
+                session.add(JobTeam(job_id=job_id, team_name=team_name))
+            session.flush()
+            job = session.get(Job, job_id)
+        job.state = JobState.RUNNING
+        return job
+
+    @provide_session
+    def _record_executor_health_result(
+        self,
+        executor_key: str,
+        team_name: str | None,
+        result: HealthCheckResult,
+        *,
+        session: Session = NEW_SESSION,
+    ) -> None:
+        """
+        Record the outcome of an executor's health check as an explicit result.
+
+        Reuses the existing Jobs table rather than introducing a new status 
model, but the result
+        itself -- not heartbeat silence -- is what ``/monitor/health`` reads 
as HEALTHY/UNHEALTHY: a
+        row that is alive but has never completed a check, or whose last check 
raised or is still
+        hanging, stays UNKNOWN instead of looking identical to a genuine 
failure.
+        """
+        job = self._get_or_create_executor_job(executor_key, team_name, 
session=session)
+        job.health_check_result = result
+        session.commit()

Review Comment:
   Same as above; @provide_session should be committing, this should not be 
required.  If it is, then something else is broken and we should find and fix 
that.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to