JoelEmmanuelCloud commented on code in PR #76410:
URL: https://github.com/apache/airflow/pull/76410#discussion_r4639209367
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -435,6 +454,41 @@ def __init__(
self.executors: list[BaseExecutor] = executors if executors else
ExecutorLoader.init_executors()
self.executor: BaseExecutor = self.executors[0]
+ # Each executor's get_health() runs on its own daemon thread, never on
the scheduler loop's own
+ # thread: a check that talks to a broker or a cluster control plane
can be slow or hang, and
+ # must not be able to delay task scheduling. A pool isn't needed since
each executor is only
+ # ever checked by one in-flight call at a time (see
_check_executor_health); a daemon thread
+ # also means a hung check can't block process exit the way a
ThreadPoolExecutor worker would.
+ self._executor_health_check_futures: dict[str, tuple[Future, float]] =
{}
+ # A probe that ignores the timeout passed to it would otherwise occupy
its slot forever: once
+ # a pending future has been sitting this many intervals, it's reported
as stalled instead of
+ # silently skipped on every later tick (see _check_executor_health).
+ self._executor_health_check_stall_after_intervals = 3
+ self._stalled_health_checks: set[str] = set()
+
+ self._executor_health_check_interval = conf.getfloat(
+ "scheduler", "executor_health_check_interval", fallback=30.0
+ )
+ # Must be comfortably below the check interval so a probe that honours
it always finishes
+ # before the next tick fires; half the interval is the starting point.
+ self._executor_health_check_timeout =
self._executor_health_check_interval / 2
+
+ # Only executors that actually override get_health() get a Job row at
all: one that was never
+ # going to report anything isn't a gap in coverage (see
_configured_executor_keys in
+ # airflow_health.py, which applies this same rule from the
/monitor/health side). An executor
+ # with no name is also skipped: _executor_health_key has nothing
stable to key its row under.
+ # getattr rather than a plain attribute access: a test double standing
in for an executor
+ # (e.g. a bare, non-speced MagicMock) has no real get_health of its
own to compare, and should
+ # be treated the same as one that doesn't override it, not raise
AttributeError.
+ from airflow.executors.base_executor import BaseExecutor as
_BaseExecutor
+
+ self._executor_health_keys: dict[str, str | None] = {
+ key: executor.team_name
+ for executor in self.executors
+ if getattr(type(executor), "get_health", _BaseExecutor.get_health)
is not _BaseExecutor.get_health
Review Comment:
moved it to a classmethod on BaseExecutor instead, same thing you're asking
about below.
used issubclass(type(executor), BaseExecutor) instead of getattr. a mock can
trick isinstance into saying yes, but calling a method on it never runs the
real code, it just hands back another mock. checking the type directly is what
actually keeps mocks out of this.
also found a real bug while poking at this: a timer was using heartrate as
its delay, and heartrate can be 0. a zero delay timer in python's sched module
just loops forever instead of returning, so this could hang the scheduler.
fixed it with a 1 second floor.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]