This is an automated email from the ASF dual-hosted git repository.
onikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8e2b198186 Scheduler job: main loop and event processing for multi
executors (#40017)
8e2b198186 is described below
commit 8e2b19818691f3a47219728979325c6736400229
Author: Niko Oliveira <[email protected]>
AuthorDate: Wed Jul 10 11:06:01 2024 -0700
Scheduler job: main loop and event processing for multi executors (#40017)
This change delivers the main portion of hybrid executors to the Airflow
scheduler. Each loop the query for TIs from the DB may contain TIs
tagged with a particular executor. We now filter those after the query
by which executors have space to take on more TIs and then sort them
into buckets per executor and send them to the appropriate executor to be
executed. We now also heartbeat all executors and process their events
each scheduler loop.
---
airflow/executors/base_executor.py | 5 +
airflow/jobs/scheduler_job_runner.py | 154 ++++++++--
tests/executors/test_base_executor.py | 28 +-
tests/jobs/test_scheduler_job.py | 558 ++++++++++++++++++++++++----------
tests/test_utils/mock_executor.py | 12 +
5 files changed, 563 insertions(+), 194 deletions(-)
diff --git a/airflow/executors/base_executor.py
b/airflow/executors/base_executor.py
index f0b232183f..153f0d3360 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -454,6 +454,11 @@ class BaseExecutor(LoggingMixin):
else:
return sys.maxsize
+ @property
+ def slots_occupied(self):
+ """Number of tasks this executor instance is currently managing."""
+ return len(self.running) + len(self.queued_tasks)
+
@staticmethod
def validate_command(command: list[str]) -> None:
"""
diff --git a/airflow/jobs/scheduler_job_runner.py
b/airflow/jobs/scheduler_job_runner.py
index 0ac822c7f2..27076d8e0d 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -84,6 +84,7 @@ if TYPE_CHECKING:
from airflow.dag_processing.manager import DagFileProcessorAgent
from airflow.executors.base_executor import BaseExecutor
+ from airflow.executors.executor_utils import ExecutorName
from airflow.models.taskinstance import TaskInstanceKey
from airflow.utils.sqlalchemy import (
CommitProhibitorGuard,
@@ -422,6 +423,15 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
task_instance_str = "\n".join(f"\t{x!r}" for x in
task_instances_to_examine)
self.log.info("%s tasks up for execution:\n%s",
len(task_instances_to_examine), task_instance_str)
+ executor_slots_available: dict[ExecutorName, int] = {}
+ # First get a mapping of executor names to slots they have
available
+ for executor in self.job.executors:
+ if TYPE_CHECKING:
+ # All executors should have a name if they are initted
from the executor_loader.
+ # But we need to check for None to make mypy happy.
+ assert executor.name
+ executor_slots_available[executor.name] =
executor.slots_available
+
for task_instance in task_instances_to_examine:
pool_name = task_instance.pool
@@ -561,6 +571,29 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
)
continue
+ if executor_obj :=
self._try_to_load_executor(task_instance.executor):
+ if TYPE_CHECKING:
+ # All executors should have a name if they are initted
from the executor_loader.
+ # But we need to check for None to make mypy happy.
+ assert executor_obj.name
+ if executor_slots_available[executor_obj.name] <= 0:
+ self.log.debug(
+ "Not scheduling %s since its executor %s does not
currently have any more "
+ "available slots"
+ )
+ starved_tasks.add((task_instance.dag_id,
task_instance.task_id))
+ continue
+ else:
+ executor_slots_available[executor_obj.name] -= 1
+ else:
+ # This is a defensive guard for if we happen to have a
task who's executor cannot be
+ # found. The check in the dag parser should make this not
realistically possible but the
+ # loader can fail if some direct DB modification has
happened or another as yet unknown
+ # edge case. _try_to_load_executor will log an error
message explaining the executor
+ # cannot be found.
+ starved_tasks.add((task_instance.dag_id,
task_instance.task_id))
+ continue
+
executable_tis.append(task_instance)
open_slots -= task_instance.pool_slots
concurrency_map.dag_active_tasks_map[dag_id] += 1
@@ -623,11 +656,14 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
make_transient(ti)
return executable_tis
- def _enqueue_task_instances_with_queued_state(self, task_instances:
list[TI], session: Session) -> None:
+ def _enqueue_task_instances_with_queued_state(
+ self, task_instances: list[TI], executor: BaseExecutor, session:
Session
+ ) -> None:
"""
Enqueue task_instances which should have been set to queued with the
executor.
:param task_instances: TaskInstances to enqueue
+ :param executor: The executor to enqueue tasks for
:param session: The session object
"""
# actually enqueue them
@@ -642,9 +678,11 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
priority = ti.priority_weight
queue = ti.queue
- self.log.info("Sending %s to executor with priority %s and queue
%s", ti.key, priority, queue)
+ self.log.info(
+ "Sending %s to %s with priority %s and queue %s", ti.key,
executor.name, priority, queue
+ )
- self.job.executor.queue_command(
+ executor.queue_command(
ti,
command,
priority=priority,
@@ -661,30 +699,53 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
2. Change the state for the TIs above atomically.
3. Enqueue the TIs in the executor.
- HA note: This function is a "critical section" meaning that only a
single executor process can execute
- this function at the same time. This is achieved by doing ``SELECT ...
from pool FOR UPDATE``. For DBs
- that support NOWAIT, a "blocked" scheduler will skip this and continue
on with other tasks (creating
- new DAG runs, progressing TIs from None to SCHEDULED etc.); DBs that
don't support this (such as
- MariaDB or MySQL 5.x) the other schedulers will wait for the lock
before continuing.
+ HA note: This function is a "critical section" meaning that only a
single scheduler process can
+ execute this function at the same time. This is achieved by doing
+ ``SELECT ... from pool FOR UPDATE``. For DBs that support NOWAIT, a
"blocked" scheduler will skip
+ this and continue on with other tasks (creating new DAG runs,
progressing TIs from None to SCHEDULED
+ etc.); DBs that don't support this (such as MariaDB or MySQL 5.x) the
other schedulers will wait for
+ the lock before continuing.
:param session:
:return: Number of task instance with state changed.
"""
+ # The user can either request a certain number of tis to schedule per
main scheduler loop (default
+ # is non-zero). If that value has been set to zero, that means use the
value of core.parallelism (or
+ # however many free slots are left). core.parallelism represents the
max number of running TIs per
+ # scheduler. Historically this value was stored in the executor, who's
job it was to control/enforce
+ # it. However, with multiple executors, any of which can run up to
core.parallelism TIs individually,
+ # we need to make sure in the scheduler now that we don't schedule
more than core.parallelism totally
+ # across all executors.
+ num_occupied_slots = sum([executor.slots_occupied for executor in
self.job.executors])
+ parallelism = conf.getint("core", "parallelism")
if self.job.max_tis_per_query == 0:
- max_tis = self.job.executor.slots_available
+ max_tis = parallelism - num_occupied_slots
else:
- max_tis = min(self.job.max_tis_per_query,
self.job.executor.slots_available)
+ max_tis = min(self.job.max_tis_per_query, parallelism -
num_occupied_slots)
+ if max_tis <= 0:
+ return 0
+
queued_tis = self._executable_task_instances_to_queued(max_tis,
session=session)
- self._enqueue_task_instances_with_queued_state(queued_tis,
session=session)
+ # Sort queued TIs to there respective executor
+ executor_to_queued_tis = self._executor_to_tis(queued_tis)
+ for executor, queued_tis_per_executor in
executor_to_queued_tis.items():
+ self.log.info(
+ "Trying to enqueue tasks: %s for executor: %s",
+ queued_tis_per_executor,
+ executor,
+ )
+
+
self._enqueue_task_instances_with_queued_state(queued_tis_per_executor,
executor, session=session)
+
return len(queued_tis)
- def _process_executor_events(self, session: Session) -> int:
+ def _process_executor_events(self, executor: BaseExecutor, session:
Session) -> int:
"""Respond to executor events."""
if not self._standalone_dag_processor and not self.processor_agent:
raise ValueError("Processor agent is not started.")
ti_primary_key_to_try_number_map: dict[tuple[str, str, str, int], int]
= {}
- event_buffer = self.job.executor.get_event_buffer()
+ event_buffer = executor.get_event_buffer()
tis_with_right_state: list[TaskInstanceKey] = []
# Report execution
@@ -725,8 +786,8 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
msg = (
"TaskInstance Finished: dag_id=%s, task_id=%s, run_id=%s,
map_index=%s, "
"run_start_date=%s, run_end_date=%s, "
- "run_duration=%s, state=%s, executor_state=%s, try_number=%s,
max_tries=%s, job_id=%s, "
- "pool=%s, queue=%s, priority_weight=%d, operator=%s,
queued_dttm=%s, "
+ "run_duration=%s, state=%s, executor=%s, executor_state=%s,
try_number=%s, max_tries=%s, "
+ "job_id=%s, pool=%s, queue=%s, priority_weight=%d,
operator=%s, queued_dttm=%s, "
"queued_by_job_id=%s, pid=%s"
)
self.log.info(
@@ -739,6 +800,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
ti.end_date,
ti.duration,
ti.state,
+ executor,
state,
try_number,
ti.max_tries,
@@ -765,7 +827,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
ti_queued = ti.try_number == buffer_key.try_number and ti.state ==
TaskInstanceState.QUEUED
ti_requeued = (
ti.queued_by_job_id != self.job.id # Another scheduler has
queued this task again
- or self.job.executor.has_task(ti) # This scheduler has this
task already
+ or executor.has_task(ti) # This scheduler has this task
already
)
if ti_queued and not ti_requeued:
@@ -774,9 +836,9 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
tags={"dag_id": ti.dag_id, "task_id": ti.task_id},
)
msg = (
- "The executor reported that the task instance %s finished
with state %s, but the task instance's state attribute is %s. " # noqa:
RUF100, UP031, flynt
+ "Executor %s reported that the task instance %s finished
with state %s, but the task instance's state attribute is %s. " # noqa:
RUF100, UP031, flynt
"Learn more:
https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally"
- % (ti, state, ti.state)
+ % (executor, ti, state, ti.state)
)
if info is not None:
msg += " Extra info: %s" % info # noqa: RUF100, UP031,
flynt
@@ -798,7 +860,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
msg=msg,
processor_subdir=ti.dag_model.processor_subdir,
)
- self.job.executor.send_callback(request)
+ executor.send_callback(request)
else:
ti.handle_failure(error=msg, session=session)
@@ -989,11 +1051,21 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
self.processor_agent.wait_until_finished()
with create_session() as session:
+ # This will schedule for as many executors as possible.
num_queued_tis = self._do_scheduling(session)
- self.job.executor.heartbeat()
+ # Heartbeat all executors, even if they're not receiving
new tasks this loop. It will be
+ # either a no-op, or they will check-in on currently
running tasks and send out new
+ # events to be processed below.
+ for executor in self.job.executors:
+ executor.heartbeat()
+
session.expunge_all()
- num_finished_events =
self._process_executor_events(session=session)
+ num_finished_events = 0
+ for executor in self.job.executors:
+ num_finished_events += self._process_executor_events(
+ executor=executor, session=session
+ )
if self.processor_agent:
self.processor_agent.heartbeat()
@@ -1092,16 +1164,18 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
session.expunge_all()
# END: schedule TIs
- if self.job.executor.slots_available <= 0:
+ # Attempt to schedule even if some executors are full but not all.
+ total_free_executor_slots = sum([executor.slots_available for
executor in self.job.executors])
+ if total_free_executor_slots <= 0:
# We know we can't do anything here, so don't even try!
- self.log.debug("Executor full, skipping critical section")
+ self.log.debug("All executors are full, skipping critical
section")
num_queued_tis = 0
else:
try:
timer = Stats.timer("scheduler.critical_section_duration")
timer.start()
- # Find anything TIs in state SCHEDULED, try to QUEUE it
(send it to the executor)
+ # Find any TIs in state SCHEDULED, try to QUEUE them (send
it to the executors)
num_queued_tis =
self._critical_section_enqueue_task_instances(session=session)
# Make sure we only sent this metric if we obtained the
lock, otherwise we'll skew the
@@ -1841,12 +1915,32 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
def _executor_to_tis(self, tis: list[TaskInstance]) -> dict[BaseExecutor,
list[TaskInstance]]:
"""Organize TIs into lists per their respective executor."""
_executor_to_tis: defaultdict[BaseExecutor, list[TaskInstance]] =
defaultdict(list)
- executor: str | None
for ti in tis:
- if ti.executor:
- executor = str(ti.executor)
- else:
- executor = None
- _executor_to_tis[ExecutorLoader.load_executor(executor)].append(ti)
+ if executor_obj := self._try_to_load_executor(ti.executor):
+ _executor_to_tis[executor_obj].append(ti)
return _executor_to_tis
+
+ def _try_to_load_executor(self, executor_name: str | None) -> BaseExecutor
| None:
+ """
+ Try to load the given executor.
+
+ In this context, we don't want to fail if the executor does not exist.
Catch the exception and
+ log to the user.
+ """
+ try:
+ return ExecutorLoader.load_executor(executor_name)
+ except ValueError as e:
+ # This case should not happen unless some (as of now unknown) edge
case occurs or direct DB
+ # modification, since the DAG parser will validate the tasks in
the DAG and ensure the executor
+ # they request is available and if not, disallow the DAG to be
scheduled.
+ # Keeping this exception handling because this is a critical issue
if we do somehow find
+ # ourselves here and the user should get some feedback about that.
+ if "Unknown executor" in str(e):
+ self.log.warning(
+ "Executor, %s, was not found but a Task was configured to
use it", executor_name
+ )
+ return None
+ else:
+ # Re-raise any other Exception not related to unknown
executors.
+ raise
diff --git a/tests/executors/test_base_executor.py
b/tests/executors/test_base_executor.py
index 0e75751faf..e6dc057ca5 100644
--- a/tests/executors/test_base_executor.py
+++ b/tests/executors/test_base_executor.py
@@ -61,6 +61,11 @@ def test_infinite_slotspool():
assert executor.slots_available == sys.maxsize
+def test_new_exec_no_slots_occupied():
+ executor = BaseExecutor(0)
+ assert executor.slots_occupied == 0
+
+
def test_get_task_log():
executor = BaseExecutor()
ti = TaskInstance(task=BaseOperator(task_id="dummy"))
@@ -108,6 +113,7 @@ def test_fail_and_success():
executor.success(key3, success_state)
assert len(executor.running) == 0
+ assert executor.slots_occupied == 0
assert len(executor.get_event_buffer()) == 3
@@ -168,9 +174,12 @@ def enqueue_tasks(executor, dagrun):
executor.queue_command(task_instance, ["airflow"])
-def setup_trigger_tasks(dag_maker):
+def setup_trigger_tasks(dag_maker, parallelism=None):
dagrun = setup_dagrun(dag_maker)
- executor = BaseExecutor()
+ if parallelism:
+ executor = BaseExecutor(parallelism=parallelism)
+ else:
+ executor = BaseExecutor()
executor.execute_async = mock.Mock()
enqueue_tasks(executor, dagrun)
return executor, dagrun
@@ -179,8 +188,21 @@ def setup_trigger_tasks(dag_maker):
@pytest.mark.db_test
@pytest.mark.parametrize("open_slots", [1, 2, 3])
def test_trigger_queued_tasks(dag_maker, open_slots):
- executor, _ = setup_trigger_tasks(dag_maker)
+ executor_parallelism = 10
+ executor, dagrun = setup_trigger_tasks(dag_maker, executor_parallelism)
+ num_tasks = len(dagrun.task_instances)
+
+ # All tasks are queued in setup method
+ assert executor.slots_occupied == num_tasks
+ assert executor.slots_available == executor_parallelism - num_tasks
+ assert len(executor.queued_tasks) == num_tasks
+ assert len(executor.running) == 0
executor.trigger_tasks(open_slots)
+ assert executor.slots_available == executor_parallelism - num_tasks
+ assert executor.slots_occupied == num_tasks
+ assert len(executor.queued_tasks) == num_tasks - open_slots
+ # Only open_slots number of tasks are allowed through to running
+ assert len(executor.running) == open_slots
assert executor.execute_async.call_count == open_slots
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index a282456c1f..3667810b4a 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -174,9 +174,53 @@ class TestSchedulerJob:
second_executor.name = MagicMock(alias="secondary_exec",
module_path="secondary.exec.module.path")
with mock.patch("airflow.jobs.job.Job.executors",
new_callable=PropertyMock) as executors_mock:
with mock.patch("airflow.jobs.job.Job.executor",
new_callable=PropertyMock) as executor_mock:
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor,
second_executor]
- yield [default_executor, second_executor]
+ with mock.patch(
+
"airflow.executors.executor_loader.ExecutorLoader.load_executor"
+ ) as loader_mock:
+ # The executors are mocked, so cannot be loaded/imported.
Mock load_executor and return
+ # the correct object for the given input executor name.
+ loader_mock.side_effect = lambda *x: {
+ (None,): default_executor,
+ ("default_exec",): default_executor,
+ ("default.exec.module.path",): default_executor,
+ ("secondary_exec",): second_executor,
+ ("secondary.exec.module.path",): second_executor,
+ }[x]
+
+ from airflow.jobs import scheduler_job_runner
+
+ # Reload the scheduler job runner so that it gets a fresh
mocked copy of the exec loader
+ reload(scheduler_job_runner)
+
+ executor_mock.return_value = default_executor
+ executors_mock.return_value = [default_executor,
second_executor]
+ yield [default_executor, second_executor]
+
+ @pytest.fixture
+ def mock_executor(self):
+ default_executor = mock.MagicMock(slots_available=8, slots_occupied=0)
+ default_executor.name = MagicMock(alias="default_exec",
module_path="default.exec.module.path")
+ with mock.patch("airflow.jobs.job.Job.executors",
new_callable=PropertyMock) as executors_mock:
+ with mock.patch("airflow.jobs.job.Job.executor",
new_callable=PropertyMock) as executor_mock:
+ with mock.patch(
+
"airflow.executors.executor_loader.ExecutorLoader.load_executor"
+ ) as loader_mock:
+ executor_mock.return_value = default_executor
+ executors_mock.return_value = [default_executor]
+ # The executor is mocked, so cannot be loaded/imported.
Mock load_executor and return the
+ # correct object for the given input executor name.
+ loader_mock.side_effect = lambda *x: {
+ ("default_exec",): default_executor,
+ ("default.exec.module.path",): default_executor,
+ (None,): default_executor,
+ }[x]
+
+ from airflow.jobs import scheduler_job_runner
+
+ # Reload the scheduler job runner so that it gets a fresh
mocked copy of the exec loader
+ reload(scheduler_job_runner)
+
+ yield default_executor
@pytest.mark.parametrize(
"configs",
@@ -274,7 +318,7 @@ class TestSchedulerJob:
executor.event_buffer[ti1.key] = State.FAILED, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.FAILED
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -286,7 +330,7 @@ class TestSchedulerJob:
session.commit()
executor.event_buffer[ti1.key] = State.SUCCESS, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.SUCCESS
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -338,7 +382,7 @@ class TestSchedulerJob:
executor.event_buffer[ti1.key] = State.FAILED, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.UP_FOR_RETRY
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -349,7 +393,7 @@ class TestSchedulerJob:
session.commit()
executor.event_buffer[ti1.key] = State.SUCCESS, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.SUCCESS
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -381,6 +425,7 @@ class TestSchedulerJob:
task_callback = mock.MagicMock()
mock_task_callback.return_value = task_callback
scheduler_job = Job(executor=executor)
+ scheduler_job.executors = [executor]
self.job_runner = SchedulerJobRunner(scheduler_job)
self.job_runner.processor_agent = mock.MagicMock()
session = settings.Session()
@@ -391,7 +436,7 @@ class TestSchedulerJob:
executor.event_buffer[ti1.key] = State.FAILED, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db()
# The state will remain in queued here and
# will be set to failed in dag parsing process
@@ -400,7 +445,7 @@ class TestSchedulerJob:
full_filepath=dag.fileloc,
simple_task_instance=mock.ANY,
processor_subdir=None,
- msg="The executor reported that the task instance "
+ msg=f"Executor {executor} reported that the task instance "
"<TaskInstance:
test_process_executor_events_with_callback.dummy_task test [queued]> "
"finished with state failed, but the task instance's state
attribute is queued. "
"Learn more:
https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally",
@@ -442,7 +487,7 @@ class TestSchedulerJob:
session.commit()
executor.event_buffer[ti1.key] = State.FAILED, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db()
assert ti1.state == State.FAILED
@@ -476,7 +521,7 @@ class TestSchedulerJob:
executor.event_buffer[ti1.key.with_try_number(1)] = State.SUCCESS, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.QUEUED
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -489,7 +534,7 @@ class TestSchedulerJob:
executor.event_buffer[ti1.key] = State.SUCCESS, None
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.QUEUED
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -503,7 +548,7 @@ class TestSchedulerJob:
executor.event_buffer[ti1.key] = State.SUCCESS, None
executor.has_task = mock.MagicMock(return_value=True)
- self.job_runner._process_executor_events(session=session)
+ self.job_runner._process_executor_events(executor=executor,
session=session)
ti1.refresh_from_db(session=session)
assert ti1.state == State.QUEUED
scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -567,16 +612,8 @@ class TestSchedulerJob:
assert isinstance(scheduler_job.executor.callback_sink,
PipeCallbackSink)
-
@mock.patch("airflow.executors.executor_loader.ExecutorLoader.init_executors")
-
@mock.patch("airflow.executors.executor_loader.ExecutorLoader.get_default_executor")
@conf_vars({("scheduler", "standalone_dag_processor"): "False"})
- def
test_setup_callback_sink_not_standalone_dag_processor_multiple_executors(
- self, get_default_executor_mock, init_executors_mock
- ):
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- init_executors_mock.return_value = [default_executor, second_executor]
- get_default_executor_mock.return_value = default_executor
+ def
test_setup_callback_sink_not_standalone_dag_processor_multiple_executors(self,
mock_executors):
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
self.job_runner._execute()
@@ -592,16 +629,8 @@ class TestSchedulerJob:
assert isinstance(scheduler_job.executor.callback_sink,
DatabaseCallbackSink)
- @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
- @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
@conf_vars({("scheduler", "standalone_dag_processor"): "True"})
- def test_setup_callback_sink_standalone_dag_processor_multiple_executors(
- self, executor_mock, executors_mock
- ):
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor, second_executor]
+ def
test_setup_callback_sink_standalone_dag_processor_multiple_executors(self,
mock_executors):
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
self.job_runner._execute()
@@ -609,14 +638,8 @@ class TestSchedulerJob:
for executor in scheduler_job.executors:
assert isinstance(executor.callback_sink, DatabaseCallbackSink)
- @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
- @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
@conf_vars({("scheduler", "standalone_dag_processor"): "True"})
- def test_executor_start_called(self, executor_mock, executors_mock):
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor, second_executor]
+ def test_executor_start_called(self, mock_executors):
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
self.job_runner._execute()
@@ -625,13 +648,7 @@ class TestSchedulerJob:
for executor in scheduler_job.executors:
executor.start.assert_called_once()
- @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
- @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
- def test_executor_job_id_assigned(self, executor_mock, executors_mock):
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor, second_executor]
+ def test_executor_job_id_assigned(self, mock_executors):
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
self.job_runner._execute()
@@ -640,13 +657,23 @@ class TestSchedulerJob:
for executor in scheduler_job.executors:
assert executor.job_id == scheduler_job.id
- @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
- @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
- def test_executor_debug_dump(self, executor_mock, executors_mock):
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor, second_executor]
+ def test_executor_heartbeat(self, mock_executors):
+ scheduler_job = Job()
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
+ self.job_runner._execute()
+
+ for executor in scheduler_job.executors:
+ executor.heartbeat.assert_called_once()
+
+ def test_executor_events_processed(self, mock_executors):
+ scheduler_job = Job()
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
+ self.job_runner._execute()
+
+ for executor in scheduler_job.executors:
+ executor.get_event_buffer.assert_called_once()
+
+ def test_executor_debug_dump(self, mock_executors):
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
self.job_runner._debug_dump(1, mock.MagicMock())
@@ -824,6 +851,45 @@ class TestSchedulerJob:
assert [ti.key for ti in res] == [tis[1].key]
session.rollback()
+ def test_find_executable_task_instances_executor(self, dag_maker,
mock_executors):
+ """
+ Test that tasks for all executors are set to queued, if space allows it
+ """
+
+ scheduler_job = Job()
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
+ session = settings.Session()
+
+ dag_id =
"SchedulerJobTest.test_find_executable_task_instances_executor"
+
+ with dag_maker(dag_id=dag_id):
+ op1 = EmptyOperator(task_id="dummy1") # No executor specified,
runs on default executor
+ op2 = EmptyOperator(task_id="dummy2", executor="default_exec")
+ op3 = EmptyOperator(task_id="dummy3",
executor="default.exec.module.path")
+ op4 = EmptyOperator(task_id="dummy4", executor="secondary_exec")
+ op5 = EmptyOperator(task_id="dummy5",
executor="secondary.exec.module.path")
+
+ dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
+
+ ti1 = dag_run.get_task_instance(op1.task_id, session)
+ ti2 = dag_run.get_task_instance(op2.task_id, session)
+ ti3 = dag_run.get_task_instance(op3.task_id, session)
+ ti4 = dag_run.get_task_instance(op4.task_id, session)
+ ti5 = dag_run.get_task_instance(op5.task_id, session)
+
+ tis_tuple = (ti1, ti2, ti3, ti4, ti5)
+ for ti in tis_tuple:
+ ti.state = State.SCHEDULED
+
+ session.flush()
+
+ res = self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
+
+ assert 5 == len(res)
+ res_ti_keys = [res_ti.key for res_ti in res]
+ for ti in tis_tuple:
+ assert ti.key in res_ti_keys
+
def test_find_executable_task_instances_order_priority_with_pools(self,
dag_maker):
"""
The scheduler job should pick tasks with higher priority for execution
@@ -892,14 +958,14 @@ class TestSchedulerJob:
assert [ti.key for ti in res] == [tis[1].key]
session.rollback()
- def test_find_executable_task_instances_in_default_pool(self, dag_maker):
+ def test_find_executable_task_instances_in_default_pool(self, dag_maker,
mock_executor):
set_default_pool_slots(1)
dag_id =
"SchedulerJobTest.test_find_executable_task_instances_in_default_pool"
with dag_maker(dag_id=dag_id):
op1 = EmptyOperator(task_id="dummy1")
op2 = EmptyOperator(task_id="dummy2")
- scheduler_job = Job(executor=MockExecutor())
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
session = settings.Session()
@@ -1193,58 +1259,60 @@ class TestSchedulerJob:
session.merge(ti2)
session.flush()
- res = self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
-
- assert 2 == len(res)
-
- ti1_1.state = State.RUNNING
- ti2.state = State.RUNNING
- ti1_2 = dr2.get_task_instance(task1.task_id)
- ti1_2.state = State.SCHEDULED
- session.merge(ti1_1)
- session.merge(ti2)
- session.merge(ti1_2)
- session.flush()
+ with
mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as
loader_mock:
+ loader_mock.side_effect = executor.get_mock_loader_side_effect()
+ res =
self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
+
+ assert 2 == len(res)
+
+ ti1_1.state = State.RUNNING
+ ti2.state = State.RUNNING
+ ti1_2 = dr2.get_task_instance(task1.task_id)
+ ti1_2.state = State.SCHEDULED
+ session.merge(ti1_1)
+ session.merge(ti2)
+ session.merge(ti1_2)
+ session.flush()
- res = self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
+ res =
self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
- assert 1 == len(res)
+ assert 1 == len(res)
- ti1_2.state = State.RUNNING
- ti1_3 = dr3.get_task_instance(task1.task_id)
- ti1_3.state = State.SCHEDULED
- session.merge(ti1_2)
- session.merge(ti1_3)
- session.flush()
+ ti1_2.state = State.RUNNING
+ ti1_3 = dr3.get_task_instance(task1.task_id)
+ ti1_3.state = State.SCHEDULED
+ session.merge(ti1_2)
+ session.merge(ti1_3)
+ session.flush()
- res = self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
+ res =
self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
- assert 0 == len(res)
+ assert 0 == len(res)
- ti1_1.state = State.SCHEDULED
- ti1_2.state = State.SCHEDULED
- ti1_3.state = State.SCHEDULED
- session.merge(ti1_1)
- session.merge(ti1_2)
- session.merge(ti1_3)
- session.flush()
+ ti1_1.state = State.SCHEDULED
+ ti1_2.state = State.SCHEDULED
+ ti1_3.state = State.SCHEDULED
+ session.merge(ti1_1)
+ session.merge(ti1_2)
+ session.merge(ti1_3)
+ session.flush()
- res = self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
+ res =
self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
- assert 2 == len(res)
+ assert 2 == len(res)
- ti1_1.state = State.RUNNING
- ti1_2.state = State.SCHEDULED
- ti1_3.state = State.SCHEDULED
- session.merge(ti1_1)
- session.merge(ti1_2)
- session.merge(ti1_3)
- session.flush()
+ ti1_1.state = State.RUNNING
+ ti1_2.state = State.SCHEDULED
+ ti1_3.state = State.SCHEDULED
+ session.merge(ti1_1)
+ session.merge(ti1_2)
+ session.merge(ti1_3)
+ session.flush()
- res = self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
+ res =
self.job_runner._executable_task_instances_to_queued(max_tis=32,
session=session)
- assert 1 == len(res)
- session.rollback()
+ assert 1 == len(res)
+ session.rollback()
def
test_change_state_for_executable_task_instances_no_tis_with_state(self,
dag_maker):
dag_id = "SchedulerJobTest.test_change_state_for__no_tis_with_state"
@@ -1533,7 +1601,9 @@ class TestSchedulerJob:
ti1 = dr1.get_task_instance(task1.task_id, session)
with patch.object(BaseExecutor, "queue_command") as mock_queue_command:
- self.job_runner._enqueue_task_instances_with_queued_state([ti1],
session=session)
+ self.job_runner._enqueue_task_instances_with_queued_state(
+ [ti1], executor=scheduler_job.executor, session=session
+ )
assert mock_queue_command.called
session.rollback()
@@ -1557,13 +1627,23 @@ class TestSchedulerJob:
session.commit()
with patch.object(BaseExecutor, "queue_command") as mock_queue_command:
- self.job_runner._enqueue_task_instances_with_queued_state([ti],
session=session)
+ self.job_runner._enqueue_task_instances_with_queued_state(
+ [ti], executor=scheduler_job.executor, session=session
+ )
session.flush()
ti.refresh_from_db(session=session)
assert ti.state == State.NONE
mock_queue_command.assert_not_called()
- def test_critical_section_enqueue_task_instances(self, dag_maker):
+ @pytest.mark.parametrize(
+ "task1_exec, task2_exec",
+ [
+ ("default_exec", "default_exec"),
+ ("default_exec", "secondary_exec"),
+ ("secondary_exec", "secondary_exec"),
+ ],
+ )
+ def test_critical_section_enqueue_task_instances(self, task1_exec,
task2_exec, dag_maker, mock_executors):
dag_id = "SchedulerJobTest.test_execute_task_instances"
task_id_1 = "dummy_task"
task_id_2 = "dummy_task_nonexistent_queue"
@@ -1573,13 +1653,13 @@ class TestSchedulerJob:
# check the num tasks once so if max_active_tasks was 3,
# we could execute arbitrarily many tasks in the second run
with dag_maker(dag_id=dag_id, max_active_tasks=3, session=session) as
dag:
- task1 = EmptyOperator(task_id=task_id_1)
- task2 = EmptyOperator(task_id=task_id_2)
+ task1 = EmptyOperator(task_id=task_id_1, executor=task1_exec)
+ task2 = EmptyOperator(task_id=task_id_2, executor=task2_exec)
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
- # create first dag run with 1 running and 1 queued
+ # create first dag run with 2 running tasks
dr1 = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
@@ -1620,7 +1700,10 @@ class TestSchedulerJob:
assert {State.QUEUED, State.SCHEDULED} == {ti3.state, ti4.state}
assert 1 == res
- def test_execute_task_instances_limit(self, dag_maker):
+ res = self.job_runner._critical_section_enqueue_task_instances(session)
+ assert 0 == res
+
+ def test_execute_task_instances_limit_second_executor(self, dag_maker,
mock_executors):
dag_id = "SchedulerJobTest.test_execute_task_instances_limit"
task_id_1 = "dummy_task"
task_id_2 = "dummy_task_2"
@@ -1630,8 +1713,8 @@ class TestSchedulerJob:
# check the num tasks once so if max_active_tasks was 3,
# we could execute arbitrarily many tasks in the second run
with dag_maker(dag_id=dag_id, max_active_tasks=16, session=session):
- task1 = EmptyOperator(task_id=task_id_1)
- task2 = EmptyOperator(task_id=task_id_2)
+ task1 = EmptyOperator(task_id=task_id_1, executor="default_exec")
+ task2 = EmptyOperator(task_id=task_id_2, executor="secondary_exec")
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
@@ -1647,33 +1730,135 @@ class TestSchedulerJob:
)
yield dagrun
- tis = []
+ tis1 = []
+ tis2 = []
for dr in _create_dagruns():
ti1 = dr.get_task_instance(task1.task_id, session)
+ tis1.append(ti1)
ti2 = dr.get_task_instance(task2.task_id, session)
+ tis2.append(ti2)
ti1.state = State.SCHEDULED
ti2.state = State.SCHEDULED
session.flush()
- scheduler_job.max_tis_per_query = 2
+ scheduler_job.max_tis_per_query = 6
+ # First pass we'll grab 6 of the 8 tasks (limited by max_tis_per_query)
+ res = self.job_runner._critical_section_enqueue_task_instances(session)
+ assert 6 == res
+ for ti in tis1[:3] + tis2[:3]:
+ ti.refresh_from_db()
+ assert ti.state == TaskInstanceState.QUEUED
+ for ti in tis1[3:] + tis2[3:]:
+ ti.refresh_from_db()
+ assert ti.state == TaskInstanceState.SCHEDULED
+
+ # The remaining TIs are queued
res = self.job_runner._critical_section_enqueue_task_instances(session)
assert 2 == res
+ for ti in tis1 + tis2:
+ ti.refresh_from_db()
+ assert State.QUEUED == ti.state
+
+ @pytest.mark.parametrize(
+ "task1_exec, task2_exec",
+ [
+ ("default_exec", "default_exec"),
+ ("default_exec", "secondary_exec"),
+ ("secondary_exec", "secondary_exec"),
+ ],
+ )
+ def test_execute_task_instances_limit(self, task1_exec, task2_exec,
dag_maker, mock_executors):
+ dag_id = "SchedulerJobTest.test_execute_task_instances_limit"
+ task_id_1 = "dummy_task"
+ task_id_2 = "dummy_task_2"
+ session = settings.Session()
+ # important that len(tasks) is less than max_active_tasks
+ # because before scheduler._execute_task_instances would only
+ # check the num tasks once so if max_active_tasks was 3,
+ # we could execute arbitrarily many tasks in the second run
+ with dag_maker(dag_id=dag_id, max_active_tasks=16, session=session):
+ task1 = EmptyOperator(task_id=task_id_1, executor=task1_exec)
+ task2 = EmptyOperator(task_id=task_id_2, executor=task2_exec)
+
+ scheduler_job = Job()
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
+
+ def _create_dagruns():
+ dagrun = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED,
state=State.RUNNING)
+ yield dagrun
+ for _ in range(3):
+ dagrun = dag_maker.create_dagrun_after(
+ dagrun,
+ run_type=DagRunType.SCHEDULED,
+ state=State.RUNNING,
+ )
+ yield dagrun
+
+ tis = []
+ for dr in _create_dagruns():
+ ti1 = dr.get_task_instance(task1.task_id, session)
+ tis.append(ti1)
+ ti2 = dr.get_task_instance(task2.task_id, session)
+ tis.append(ti2)
+ ti1.state = State.SCHEDULED
+ ti2.state = State.SCHEDULED
+ session.flush()
+ scheduler_job.max_tis_per_query = 2
+
+ total_enqueued =
self.job_runner._critical_section_enqueue_task_instances(session)
+ assert 2 == total_enqueued
+
+ def test_execute_task_instances_limit_slots(self, dag_maker,
mock_executors):
+ dag_id = "SchedulerJobTest.test_execute_task_instances_limit"
+ task_id_1 = "dummy_task"
+ task_id_2 = "dummy_task_2"
+ session = settings.Session()
+ # important that len(tasks) is less than max_active_tasks
+ # because before scheduler._execute_task_instances would only
+ # check the num tasks once so if max_active_tasks was 3,
+ # we could execute arbitrarily many tasks in the second run
+ with dag_maker(dag_id=dag_id, max_active_tasks=16, session=session):
+ task1 = EmptyOperator(task_id=task_id_1, executor="default_exec")
+ task2 = EmptyOperator(task_id=task_id_2, executor="secondary_exec")
+
+ scheduler_job = Job()
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
+
+ def _create_dagruns():
+ dagrun = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED,
state=State.RUNNING)
+ yield dagrun
+ for _ in range(3):
+ dagrun = dag_maker.create_dagrun_after(
+ dagrun,
+ run_type=DagRunType.SCHEDULED,
+ state=State.RUNNING,
+ )
+ yield dagrun
+
+ tis = []
+ for dr in _create_dagruns():
+ ti1 = dr.get_task_instance(task1.task_id, session)
+ tis.append(ti1)
+ ti2 = dr.get_task_instance(task2.task_id, session)
+ tis.append(ti2)
+ ti1.state = State.SCHEDULED
+ ti2.state = State.SCHEDULED
+ session.flush()
+
scheduler_job.max_tis_per_query = 8
- with mock.patch.object(
- type(scheduler_job.executor), "slots_available",
new_callable=mock.PropertyMock
- ) as mock_slots:
- mock_slots.return_value = 2
- # Check that we don't "overfill" the executor
- assert 2 == res
- res =
self.job_runner._critical_section_enqueue_task_instances(session)
+ scheduler_job.executor.slots_available = 2 # Limit only the default
executor to 2 slots.
+ # Check that we don't "overfill" an executor when the max tis per
query is larger than slots
+ # available. Of the 8 tasks returned by the query, the default
executor will only take 2 and the
+ # secondary executor will take 4 (since only 4 of the 8 TIs in the
result will be for that executor)
+ res = self.job_runner._critical_section_enqueue_task_instances(session)
+ assert res == 6
+ scheduler_job.executor.slots_available = 6 # The default executor has
more slots freed now and
+ # will take the other two TIs.
res = self.job_runner._critical_section_enqueue_task_instances(session)
- assert 4 == res
- for ti in tis:
- ti.refresh_from_db()
- assert State.QUEUED == ti.state
+ assert res == 2
- def test_execute_task_instances_unlimited(self, dag_maker):
+ def test_execute_task_instances_unlimited(self, dag_maker, mock_executor):
"""Test that max_tis_per_query=0 is unlimited"""
dag_id = "SchedulerJobTest.test_execute_task_instances_unlimited"
@@ -1706,11 +1891,73 @@ class TestSchedulerJob:
ti2.state = State.SCHEDULED
session.flush()
scheduler_job.max_tis_per_query = 0
- scheduler_job.executor = MagicMock(slots_available=36)
+ scheduler_job.executor.parallelism = 32
+ scheduler_job.executor.slots_available = 31
res = self.job_runner._critical_section_enqueue_task_instances(session)
# 20 dag runs * 2 tasks each = 40, but limited by number of slots
available
- assert res == 36
+ assert res == 31
+ session.rollback()
+
+ @pytest.mark.parametrize(
+ "task1_exec, task2_exec",
+ [
+ ("default_exec", "default_exec"),
+ ("default_exec", "secondary_exec"),
+ ("secondary_exec", "secondary_exec"),
+ ],
+ )
+ def test_execute_task_instances_unlimited_multiple_executors(
+ self, task1_exec, task2_exec, dag_maker, mock_executors
+ ):
+ """Test that max_tis_per_query=0 is unlimited"""
+
+ dag_id = "SchedulerJobTest.test_execute_task_instances_unlimited"
+ task_id_1 = "dummy_task"
+ task_id_2 = "dummy_task_2"
+ session = settings.Session()
+
+ with dag_maker(dag_id=dag_id, max_active_tasks=1024, session=session):
+ task1 = EmptyOperator(task_id=task_id_1, executor=task1_exec)
+ task2 = EmptyOperator(task_id=task_id_2, executor=task2_exec)
+
+ scheduler_job = Job()
+ self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
+
+ def _create_dagruns():
+ dagrun = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED,
state=State.RUNNING)
+ yield dagrun
+ for _ in range(40):
+ dagrun = dag_maker.create_dagrun_after(
+ dagrun,
+ run_type=DagRunType.SCHEDULED,
+ state=State.RUNNING,
+ )
+ yield dagrun
+
+ for dr in _create_dagruns():
+ ti1 = dr.get_task_instance(task1.task_id, session)
+ ti2 = dr.get_task_instance(task2.task_id, session)
+ ti1.state = State.SCHEDULED
+ ti2.state = State.SCHEDULED
+ session.flush()
+ scheduler_job.max_tis_per_query = 0
+ for executor in mock_executors:
+ executor.parallelism = 32
+ executor.slots_available = 31
+
+ total_enqueued = 0
+ with conf_vars({("core", "parallelism"): "40"}):
+ # 40 dag runs * 2 tasks each = 80. Two executors have capacity for
61 concurrent jobs, but they
+ # together respect core.parallelism and will not run more in
aggregate then that allows.
+ total_enqueued +=
self.job_runner._critical_section_enqueue_task_instances(session)
+
+ if task1_exec != task2_exec:
+ # Two executors will execute up to core parallelism
+ assert total_enqueued == 40
+ else:
+ # A single executor will only run up to its available slots
+ assert total_enqueued == 31
session.rollback()
def test_adopt_or_reset_orphaned_tasks(self, dag_maker):
@@ -1864,33 +2111,22 @@ class TestSchedulerJob:
assert "Executor doesn't support cleanup of stuck queued tasks.
Skipping." in caplog.text
@mock.patch("airflow.dag_processing.manager.DagFileProcessorAgent")
- def test_executor_end_called(self, mock_processor_agent):
+ def test_executor_end_called(self, mock_processor_agent, mock_executors):
"""
Test to make sure executor.end gets called with a successful scheduler
loop run
"""
- scheduler_job = Job(executor=mock.MagicMock(slots_available=8))
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
run_job(scheduler_job, execute_callable=self.job_runner._execute)
scheduler_job.executor.end.assert_called_once()
self.job_runner.processor_agent.end.assert_called_once()
- @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
- @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
@mock.patch("airflow.dag_processing.manager.DagFileProcessorAgent")
- def test_executor_end_called_multiple_executors(
- self, mock_processor_agent, executor_mock, executors_mock
- ):
+ def test_executor_end_called_multiple_executors(self,
mock_processor_agent, mock_executors):
"""
Test to make sure executor.end gets called on all executors with a
successful scheduler loop run
"""
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor, second_executor]
scheduler_job = Job()
- assert scheduler_job.executor is default_executor
- assert scheduler_job.executors == [default_executor, second_executor]
-
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
run_job(scheduler_job, execute_callable=self.job_runner._execute)
scheduler_job.executor.end.assert_called_once()
@@ -1917,22 +2153,13 @@ class TestSchedulerJob:
scheduler_job.executor.end.assert_called_once()
mock_processor_agent.return_value.end.reset_mock(side_effect=True)
- @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
- @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
@mock.patch("airflow.dag_processing.manager.DagFileProcessorAgent")
- def test_cleanup_methods_all_called_multiple_executors(
- self, mock_processor_agent, executor_mock, executors_mock
- ):
+ def test_cleanup_methods_all_called_multiple_executors(self,
mock_processor_agent, mock_executors):
"""
Test to make sure all cleanup methods are called when the scheduler
loop has an exception
"""
- default_executor = mock.MagicMock(slots_available=8)
- second_executor = mock.MagicMock(slots_available=8)
- executor_mock.return_value = default_executor
- executors_mock.return_value = [default_executor, second_executor]
scheduler_job = Job()
- assert scheduler_job.executor is default_executor
- assert scheduler_job.executors == [default_executor, second_executor]
+
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull, num_runs=1)
self.job_runner._run_scheduler_loop =
mock.MagicMock(side_effect=RuntimeError("oops"))
mock_processor_agent.return_value.end.side_effect =
RuntimeError("double oops")
@@ -2604,7 +2831,7 @@ class TestSchedulerJob:
dr.update_state()
assert dr.state == State.RUNNING
- def test_dagrun_root_after_dagrun_unfinished(self):
+ def test_dagrun_root_after_dagrun_unfinished(self, mock_executor):
"""
DagRuns with one successful and one future root task -> SUCCESS
@@ -2615,7 +2842,7 @@ class TestSchedulerJob:
dag = self.dagbag.get_dag(dag_id)
dag.sync_to_db()
- scheduler_job = Job(executor=self.null_exec)
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job, num_runs=1,
subdir=dag.fileloc)
run_job(scheduler_job, execute_callable=self.job_runner._execute)
@@ -2809,7 +3036,7 @@ class TestSchedulerJob:
{("scheduler", "standalone_dag_processor"): "True"},
],
)
- def test_scheduler_verify_pool_full(self, dag_maker, configs):
+ def test_scheduler_verify_pool_full(self, dag_maker, configs,
mock_executor):
"""
Test task instances not queued when pool is full
"""
@@ -2826,7 +3053,7 @@ class TestSchedulerJob:
session.add(pool)
session.flush()
- scheduler_job = Job(executor=self.null_exec)
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job)
self.job_runner.processor_agent = mock.MagicMock()
@@ -2845,7 +3072,7 @@ class TestSchedulerJob:
assert len(task_instances_list) == 1
@pytest.mark.need_serialized_dag
- def test_scheduler_verify_pool_full_2_slots_per_task(self, dag_maker,
session):
+ def test_scheduler_verify_pool_full_2_slots_per_task(self, dag_maker,
session, mock_executor):
"""
Test task instances not queued when pool is full.
@@ -2867,7 +3094,7 @@ class TestSchedulerJob:
session.add(pool)
session.flush()
- scheduler_job = Job(executor=self.null_exec)
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job)
self.job_runner.processor_agent = mock.MagicMock()
@@ -2890,7 +3117,7 @@ class TestSchedulerJob:
# As tasks require 2 slots, only 3 can fit into 6 available
assert len(task_instances_list) == 3
- def test_scheduler_keeps_scheduling_pool_full(self, dag_maker):
+ def test_scheduler_keeps_scheduling_pool_full(self, dag_maker,
mock_executor):
"""
Test task instances in a pool that isn't full keep getting scheduled
even when a pool is full.
"""
@@ -2923,7 +3150,7 @@ class TestSchedulerJob:
session.add(pool_p2)
session.flush()
- scheduler_job = Job(executor=self.null_exec)
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job)
self.job_runner.processor_agent = mock.MagicMock()
@@ -2961,7 +3188,7 @@ class TestSchedulerJob:
for task_instance in task_instances_list2
)
- def test_scheduler_verify_priority_and_slots(self, dag_maker):
+ def test_scheduler_verify_priority_and_slots(self, dag_maker,
mock_executor):
"""
Test task instances with higher priority are not queued
when pool does not have enough slots.
@@ -2999,7 +3226,7 @@ class TestSchedulerJob:
session.add(pool)
session.flush()
- scheduler_job = Job(executor=self.null_exec)
+ scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job)
self.job_runner.processor_agent = mock.MagicMock()
@@ -3239,7 +3466,11 @@ class TestSchedulerJob:
# Since the DAG is not in the directory watched by scheduler job,
# it would've been marked as deleted and not being scheduled.
with mock.patch.object(DagModel, "deactivate_deleted_dags"):
- run_job(scheduler_job,
execute_callable=self.job_runner._execute)
+ with mock.patch(
+
"airflow.executors.executor_loader.ExecutorLoader.load_executor"
+ ) as loader_mock:
+ loader_mock.side_effect =
executor.get_mock_loader_side_effect()
+ run_job(scheduler_job,
execute_callable=self.job_runner._execute)
do_schedule()
with create_session() as session:
@@ -4221,10 +4452,16 @@ class TestSchedulerJob:
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
- scheduler_job.executor = MockExecutor(do_update=False)
- self.job_runner.processor_agent =
mock.MagicMock(spec=DagFileProcessorAgent)
+ executor = MockExecutor(do_update=False)
+ scheduler_job.executor = executor
+ scheduler_job.executors = [executor]
+ with
mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as
loader_mock:
+ # The executor is mocked, so cannot be loaded/imported. Mock
load_executor and return the
+ # correct object for the given input executor name.
+ loader_mock.side_effect = executor.get_mock_loader_side_effect()
+ self.job_runner.processor_agent =
mock.MagicMock(spec=DagFileProcessorAgent)
- num_queued = self.job_runner._do_scheduling(session)
+ num_queued = self.job_runner._do_scheduling(session)
assert num_queued == 1
session.flush()
@@ -4339,7 +4576,7 @@ class TestSchedulerJob:
]
assert dagrun_execution_dates == expected_execution_dates
- def test_do_schedule_max_active_runs_and_manual_trigger(self, dag_maker):
+ def test_do_schedule_max_active_runs_and_manual_trigger(self, dag_maker,
mock_executors):
"""
Make sure that when a DAG is already at max_active_runs, that manually
triggered
dagruns don't start running.
@@ -4369,7 +4606,6 @@ class TestSchedulerJob:
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job,
subdir=os.devnull)
- scheduler_job.executor = MockExecutor(do_update=False)
self.job_runner.processor_agent =
mock.MagicMock(spec=DagFileProcessorAgent)
num_queued = self.job_runner._do_scheduling(session)
diff --git a/tests/test_utils/mock_executor.py
b/tests/test_utils/mock_executor.py
index a02af122bf..5080290500 100644
--- a/tests/test_utils/mock_executor.py
+++ b/tests/test_utils/mock_executor.py
@@ -21,6 +21,7 @@ from collections import defaultdict
from unittest.mock import MagicMock, Mock
from airflow.executors.base_executor import BaseExecutor
+from airflow.executors.executor_utils import ExecutorName
from airflow.models.taskinstancekey import TaskInstanceKey
from airflow.utils.session import create_session
from airflow.utils.state import State
@@ -32,6 +33,8 @@ class MockExecutor(BaseExecutor):
"""
supports_pickling = False
+ mock_module_path = "mock.executor.path"
+ mock_alias = "mock_executor"
def __init__(self, do_update=True, *args, **kwargs):
self.do_update = do_update
@@ -43,6 +46,8 @@ class MockExecutor(BaseExecutor):
# All the tasks, in a stable sort order
self.sorted_tasks = []
+ self.name = ExecutorName(module_path=self.mock_module_path,
alias=self.mock_alias)
+
# If multiprocessing runs in spawn mode,
# arguments are to be pickled but lambda is not picclable.
# So we should pass self.success instead of lambda.
@@ -101,3 +106,10 @@ class MockExecutor(BaseExecutor):
"""
assert isinstance(run_id, str)
self.mock_task_results[TaskInstanceKey(dag_id, task_id, run_id,
try_number)] = State.FAILED
+
+ def get_mock_loader_side_effect(self):
+ return lambda *x: {
+ (None,): self,
+ (self.mock_module_path,): self,
+ (self.mock_alias,): self,
+ }[x]