ferruzzi commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4633324255
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -456,6 +510,184 @@ def __init__(
def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
stats.incr("scheduler_heartbeat", 1, 1)
+ def _check_executor_health(self) -> None:
+ """
+ Kick off a health check for each reporting executor, in the background.
+
+ Called on its own timer (``[scheduler]
executor_health_check_interval``), independent of the
+ scheduling loop and of each executor's own
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+ A new check for a given executor is skipped while its previous check
is still running, so a
+ slow or hung dependency can only ever delay that executor's own health
being updated, never
+ pile up work or block this (or any other) scheduler tick.
+
+ ``get_health()`` is passed a timeout, but an implementation that
ignores it can still leave its
+ future pending forever. Once a pending future is a few intervals old,
it's reported as stalled:
+ the scheduler records UNHEALTHY itself, since a check that hasn't
answered isn't still-healthy,
+ and logs and emits a metric so the stall is visible. If the stuck call
does eventually return,
+ its result overwrites this one.
+ """
+ for executor in self.executors:
+ key = _executor_health_key(executor)
+ if key is None or key not in self._executor_health_keys:
+ # No name, or doesn't override get_health(): there's nothing
to check or record.
+ continue
+ future, started = self._executor_health_check_futures.get(key,
(None, 0.0))
+ if future is not None and not future.done():
+ stall_after = (
+ self._executor_health_check_interval *
self._executor_health_check_stall_after_intervals
+ )
+ if time.monotonic() - started > stall_after and key not in
self._stalled_health_checks:
+ self._stalled_health_checks.add(key)
+ self.log.warning("Health check for executor %s has stopped
responding", key)
+ stats.incr(
+ "scheduler.executor_health_check.stalled",
+ tags=prune_dict(
+ {"executor": type(executor).__name__, "team_name":
executor.team_name}
+ ),
+ )
+ try:
+ self._record_executor_health_result(
+ key, executor.team_name,
HealthCheckResult.UNHEALTHY
+ )
+ except Exception:
+ self.log.exception("Failed to record health check
result for executor %s", key)
+ else:
+ self.log.debug(
+ "Skipping health check for executor %s, the previous
check hasn't finished", key
+ )
+ continue
+ self._stalled_health_checks.discard(key)
+ future = Future()
+ threading.Thread(
+ target=self._run_executor_health_check_and_resolve,
+ args=(executor, key, future),
+ name=f"executor-health-check-{key}",
+ daemon=True,
+ ).start()
+ self._executor_health_check_futures[key] = (future,
time.monotonic())
+
+ def _run_executor_health_check_and_resolve(
+ self, executor: BaseExecutor, key: str, future: Future
+ ) -> None:
+ """Run one executor's health check and resolve its tracking future,
even if it raises."""
+ try:
+ self._run_executor_health_check(executor, key)
+ finally:
+ future.set_result(None)
+
+ def _run_executor_health_check(self, executor: BaseExecutor, key: str) ->
None:
+ """Run one executor's get_health() off the scheduler thread and record
an explicit result."""
+ try:
+ healthy =
executor.get_health(timeout=self._executor_health_check_timeout)
+ except NotImplementedError:
+ healthy = None
+ except Exception:
+ self.log.exception("Health check for executor %s raised an
exception", key)
+ healthy = False
+ if healthy is None:
+ result = HealthCheckResult.UNKNOWN
+ elif healthy:
+ result = HealthCheckResult.HEALTHY
+ else:
+ self.log.warning("Executor %s reported itself unhealthy", key)
+ result = HealthCheckResult.UNHEALTHY
+ try:
+ self._record_executor_health_result(key, executor.team_name,
result)
+ except Exception:
+ self.log.exception("Failed to record health check result for
executor %s", key)
+
+ @provide_session
+ def _touch_executor_heartbeats(self, *, session: Session = NEW_SESSION) ->
None:
+ """
+ Keep every reporting executor's Job row heartbeat moving on the
scheduler's own cadence.
+
+ ``get_health()`` itself only runs every
``executor_health_check_interval`` (default 30s), far
+ slower than a Job row can go quiet for before
``health_check_threshold`` calls it dead.
+ Updating the heartbeat only when a check finishes would make a
perfectly healthy executor
+ flicker to UNHEALTHY between checks, every cycle. Keeping the
heartbeat on the scheduler's own
+ short cadence instead, the same way it already does for its own Job
row, means the row's
+ liveness reflects "the scheduler is still running and still intends to
check this executor",
+ while ``health_check_result`` -- not this heartbeat -- carries
whatever the last actual check
+ found.
+ """
+ if not self._executor_health_keys:
+ return
+ now = timezone.utcnow()
+ for key, team_name in self._executor_health_keys.items():
+ job = self._get_or_create_executor_job(key, team_name,
session=session)
+ job.latest_heartbeat = now
+ session.commit()
Review Comment:
A method which uses the @provide_session decorator shouldn't have to do
session.commit(), if this is required then something else is going on. the
decorator should be commiting the session when it exits. Try removing it and
see if it still works.
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -456,6 +510,184 @@ def __init__(
def heartbeat_callback(self, *, session: Session = NEW_SESSION) -> None:
stats.incr("scheduler_heartbeat", 1, 1)
+ def _check_executor_health(self) -> None:
+ """
+ Kick off a health check for each reporting executor, in the background.
+
+ Called on its own timer (``[scheduler]
executor_health_check_interval``), independent of the
+ scheduling loop and of each executor's own
:meth:`~airflow.executors.base_executor.BaseExecutor.heartbeat`.
+ A new check for a given executor is skipped while its previous check
is still running, so a
+ slow or hung dependency can only ever delay that executor's own health
being updated, never
+ pile up work or block this (or any other) scheduler tick.
+
+ ``get_health()`` is passed a timeout, but an implementation that
ignores it can still leave its
+ future pending forever. Once a pending future is a few intervals old,
it's reported as stalled:
+ the scheduler records UNHEALTHY itself, since a check that hasn't
answered isn't still-healthy,
+ and logs and emits a metric so the stall is visible. If the stuck call
does eventually return,
+ its result overwrites this one.
+ """
+ for executor in self.executors:
+ key = _executor_health_key(executor)
+ if key is None or key not in self._executor_health_keys:
+ # No name, or doesn't override get_health(): there's nothing
to check or record.
+ continue
+ future, started = self._executor_health_check_futures.get(key,
(None, 0.0))
+ if future is not None and not future.done():
+ stall_after = (
+ self._executor_health_check_interval *
self._executor_health_check_stall_after_intervals
+ )
+ if time.monotonic() - started > stall_after and key not in
self._stalled_health_checks:
+ self._stalled_health_checks.add(key)
+ self.log.warning("Health check for executor %s has stopped
responding", key)
+ stats.incr(
+ "scheduler.executor_health_check.stalled",
+ tags=prune_dict(
+ {"executor": type(executor).__name__, "team_name":
executor.team_name}
+ ),
+ )
+ try:
+ self._record_executor_health_result(
+ key, executor.team_name,
HealthCheckResult.UNHEALTHY
+ )
+ except Exception:
+ self.log.exception("Failed to record health check
result for executor %s", key)
+ else:
+ self.log.debug(
+ "Skipping health check for executor %s, the previous
check hasn't finished", key
+ )
+ continue
+ self._stalled_health_checks.discard(key)
+ future = Future()
+ threading.Thread(
+ target=self._run_executor_health_check_and_resolve,
+ args=(executor, key, future),
+ name=f"executor-health-check-{key}",
+ daemon=True,
+ ).start()
+ self._executor_health_check_futures[key] = (future,
time.monotonic())
+
+ def _run_executor_health_check_and_resolve(
+ self, executor: BaseExecutor, key: str, future: Future
+ ) -> None:
+ """Run one executor's health check and resolve its tracking future,
even if it raises."""
+ try:
+ self._run_executor_health_check(executor, key)
+ finally:
+ future.set_result(None)
+
+ def _run_executor_health_check(self, executor: BaseExecutor, key: str) ->
None:
+ """Run one executor's get_health() off the scheduler thread and record
an explicit result."""
+ try:
+ healthy =
executor.get_health(timeout=self._executor_health_check_timeout)
+ except NotImplementedError:
+ healthy = None
+ except Exception:
+ self.log.exception("Health check for executor %s raised an
exception", key)
+ healthy = False
+ if healthy is None:
+ result = HealthCheckResult.UNKNOWN
+ elif healthy:
+ result = HealthCheckResult.HEALTHY
+ else:
+ self.log.warning("Executor %s reported itself unhealthy", key)
+ result = HealthCheckResult.UNHEALTHY
+ try:
+ self._record_executor_health_result(key, executor.team_name,
result)
+ except Exception:
+ self.log.exception("Failed to record health check result for
executor %s", key)
+
+ @provide_session
+ def _touch_executor_heartbeats(self, *, session: Session = NEW_SESSION) ->
None:
+ """
+ Keep every reporting executor's Job row heartbeat moving on the
scheduler's own cadence.
+
+ ``get_health()`` itself only runs every
``executor_health_check_interval`` (default 30s), far
+ slower than a Job row can go quiet for before
``health_check_threshold`` calls it dead.
+ Updating the heartbeat only when a check finishes would make a
perfectly healthy executor
+ flicker to UNHEALTHY between checks, every cycle. Keeping the
heartbeat on the scheduler's own
+ short cadence instead, the same way it already does for its own Job
row, means the row's
+ liveness reflects "the scheduler is still running and still intends to
check this executor",
+ while ``health_check_result`` -- not this heartbeat -- carries
whatever the last actual check
+ found.
+ """
+ if not self._executor_health_keys:
+ return
+ now = timezone.utcnow()
+ for key, team_name in self._executor_health_keys.items():
+ job = self._get_or_create_executor_job(key, team_name,
session=session)
+ job.latest_heartbeat = now
+ session.commit()
+
+ def _get_or_create_executor_job(
+ self, executor_key: str, team_name: str | None, *, session: Session
+ ) -> Job:
+ """
+ Return this scheduler's own Job row for ``executor_key``, creating it
if needed.
+
+ Keyed by ``owner_job_id`` (this scheduler's own Job id), not just
``hostname``: two schedulers
+ running the same executor against shared infrastructure (e.g. the same
Celery broker) each get
+ their own row, so one scheduler's healthy report can never paper over
another's failing one.
+ A restarted scheduler gets a fresh row for the same reason its own Job
row is fresh -- a new
+ id -- instead of quietly taking over a row left behind by its previous
instance.
+
+ Inserted directly rather than through ``Job(...)``: that constructor
fires the ``on_starting``
+ listener hook, meant for "a component is starting", but the scheduler
is already running and
+ nothing here ever fires the matching ``before_stopping``. A plain
insert sidesteps that, the
+ same way the Execution API's job-registration endpoint does for Jobs
it creates on another
+ component's behalf.
+ """
+ job = session.scalar(
+ select(Job).where(
+ Job.job_type == EXECUTOR_JOB_TYPE,
+ Job.executor_class == executor_key,
+ Job.owner_job_id == self.job.id,
+ Job.end_date.is_(None),
+ )
+ )
+ if job is None:
+ now = timezone.utcnow()
+ inserted = session.execute(
+ insert(Job).values(
+ job_type=EXECUTOR_JOB_TYPE,
+ executor_class=executor_key,
+ owner_job_id=self.job.id,
+ health_check_result=HealthCheckResult.UNKNOWN,
+ state=JobState.RUNNING,
+ start_date=now,
+ latest_heartbeat=now,
+ hostname=get_hostname(),
+ unixname=getuser(),
+ )
+ )
+ job_id = cast("CursorResult", inserted).inserted_primary_key[0] #
type: ignore[index]
+ if team_name:
+ session.add(JobTeam(job_id=job_id, team_name=team_name))
+ session.flush()
+ job = session.get(Job, job_id)
+ job.state = JobState.RUNNING
+ return job
+
+ @provide_session
+ def _record_executor_health_result(
+ self,
+ executor_key: str,
+ team_name: str | None,
+ result: HealthCheckResult,
+ *,
+ session: Session = NEW_SESSION,
+ ) -> None:
+ """
+ Record the outcome of an executor's health check as an explicit result.
+
+ Reuses the existing Jobs table rather than introducing a new status
model, but the result
+ itself -- not heartbeat silence -- is what ``/monitor/health`` reads
as HEALTHY/UNHEALTHY: a
+ row that is alive but has never completed a check, or whose last check
raised or is still
+ hanging, stays UNKNOWN instead of looking identical to a genuine
failure.
+ """
+ job = self._get_or_create_executor_job(executor_key, team_name,
session=session)
+ job.health_check_result = result
+ session.commit()
Review Comment:
Same as above; @provide_session should be committing, this should not be
required. If it is, then something else is broken and we should find and fix
that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]