This is an automated email from the ASF dual-hosted git repository.

onikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 8e2b198186 Scheduler job: main loop and event processing for multi 
executors (#40017)
8e2b198186 is described below

commit 8e2b19818691f3a47219728979325c6736400229
Author: Niko Oliveira <[email protected]>
AuthorDate: Wed Jul 10 11:06:01 2024 -0700

    Scheduler job: main loop and event processing for multi executors (#40017)
    
    This change delivers the main portion of hybrid executors to the Airflow
    scheduler. Each loop the query for TIs from the DB may contain TIs
    tagged with a particular executor. We now filter those after the query
    by which executors have space to take on more TIs and then sort them
    into buckets per executor and send them to the appropriate executor to be
    executed. We now also heartbeat all executors and process their events
    each scheduler loop.
---
 airflow/executors/base_executor.py    |   5 +
 airflow/jobs/scheduler_job_runner.py  | 154 ++++++++--
 tests/executors/test_base_executor.py |  28 +-
 tests/jobs/test_scheduler_job.py      | 558 ++++++++++++++++++++++++----------
 tests/test_utils/mock_executor.py     |  12 +
 5 files changed, 563 insertions(+), 194 deletions(-)

diff --git a/airflow/executors/base_executor.py 
b/airflow/executors/base_executor.py
index f0b232183f..153f0d3360 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -454,6 +454,11 @@ class BaseExecutor(LoggingMixin):
         else:
             return sys.maxsize
 
+    @property
+    def slots_occupied(self):
+        """Number of tasks this executor instance is currently managing."""
+        return len(self.running) + len(self.queued_tasks)
+
     @staticmethod
     def validate_command(command: list[str]) -> None:
         """
diff --git a/airflow/jobs/scheduler_job_runner.py 
b/airflow/jobs/scheduler_job_runner.py
index 0ac822c7f2..27076d8e0d 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -84,6 +84,7 @@ if TYPE_CHECKING:
 
     from airflow.dag_processing.manager import DagFileProcessorAgent
     from airflow.executors.base_executor import BaseExecutor
+    from airflow.executors.executor_utils import ExecutorName
     from airflow.models.taskinstance import TaskInstanceKey
     from airflow.utils.sqlalchemy import (
         CommitProhibitorGuard,
@@ -422,6 +423,15 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             task_instance_str = "\n".join(f"\t{x!r}" for x in 
task_instances_to_examine)
             self.log.info("%s tasks up for execution:\n%s", 
len(task_instances_to_examine), task_instance_str)
 
+            executor_slots_available: dict[ExecutorName, int] = {}
+            # First get a mapping of executor names to slots they have 
available
+            for executor in self.job.executors:
+                if TYPE_CHECKING:
+                    # All executors should have a name if they are initted 
from the executor_loader.
+                    # But we need to check for None to make mypy happy.
+                    assert executor.name
+                executor_slots_available[executor.name] = 
executor.slots_available
+
             for task_instance in task_instances_to_examine:
                 pool_name = task_instance.pool
 
@@ -561,6 +571,29 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
                             )
                             continue
 
+                if executor_obj := 
self._try_to_load_executor(task_instance.executor):
+                    if TYPE_CHECKING:
+                        # All executors should have a name if they are initted 
from the executor_loader.
+                        # But we need to check for None to make mypy happy.
+                        assert executor_obj.name
+                    if executor_slots_available[executor_obj.name] <= 0:
+                        self.log.debug(
+                            "Not scheduling %s since its executor %s does not 
currently have any more "
+                            "available slots"
+                        )
+                        starved_tasks.add((task_instance.dag_id, 
task_instance.task_id))
+                        continue
+                    else:
+                        executor_slots_available[executor_obj.name] -= 1
+                else:
+                    # This is a defensive guard for if we happen to have a 
task who's executor cannot be
+                    # found. The check in the dag parser should make this not 
realistically possible but the
+                    # loader can fail if some direct DB modification has 
happened or another as yet unknown
+                    # edge case. _try_to_load_executor will log an error 
message explaining the executor
+                    # cannot be found.
+                    starved_tasks.add((task_instance.dag_id, 
task_instance.task_id))
+                    continue
+
                 executable_tis.append(task_instance)
                 open_slots -= task_instance.pool_slots
                 concurrency_map.dag_active_tasks_map[dag_id] += 1
@@ -623,11 +656,14 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             make_transient(ti)
         return executable_tis
 
-    def _enqueue_task_instances_with_queued_state(self, task_instances: 
list[TI], session: Session) -> None:
+    def _enqueue_task_instances_with_queued_state(
+        self, task_instances: list[TI], executor: BaseExecutor, session: 
Session
+    ) -> None:
         """
         Enqueue task_instances which should have been set to queued with the 
executor.
 
         :param task_instances: TaskInstances to enqueue
+        :param executor: The executor to enqueue tasks for
         :param session: The session object
         """
         # actually enqueue them
@@ -642,9 +678,11 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
 
             priority = ti.priority_weight
             queue = ti.queue
-            self.log.info("Sending %s to executor with priority %s and queue 
%s", ti.key, priority, queue)
+            self.log.info(
+                "Sending %s to %s with priority %s and queue %s", ti.key, 
executor.name, priority, queue
+            )
 
-            self.job.executor.queue_command(
+            executor.queue_command(
                 ti,
                 command,
                 priority=priority,
@@ -661,30 +699,53 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
         2. Change the state for the TIs above atomically.
         3. Enqueue the TIs in the executor.
 
-        HA note: This function is a "critical section" meaning that only a 
single executor process can execute
-        this function at the same time. This is achieved by doing ``SELECT ... 
from pool FOR UPDATE``. For DBs
-        that support NOWAIT, a "blocked" scheduler will skip this and continue 
on with other tasks (creating
-        new DAG runs, progressing TIs from None to SCHEDULED etc.); DBs that 
don't support this (such as
-        MariaDB or MySQL 5.x) the other schedulers will wait for the lock 
before continuing.
+        HA note: This function is a "critical section" meaning that only a 
single scheduler process can
+        execute this function at the same time. This is achieved by doing
+        ``SELECT ... from pool FOR UPDATE``. For DBs that support NOWAIT, a 
"blocked" scheduler will skip
+        this and continue on with other tasks (creating new DAG runs, 
progressing TIs from None to SCHEDULED
+        etc.); DBs that don't support this (such as MariaDB or MySQL 5.x) the 
other schedulers will wait for
+        the lock before continuing.
 
         :param session:
         :return: Number of task instance with state changed.
         """
+        # The user can either request a certain number of tis to schedule per 
main scheduler loop (default
+        # is non-zero). If that value has been set to zero, that means use the 
value of core.parallelism (or
+        # however many free slots are left). core.parallelism represents the 
max number of running TIs per
+        # scheduler. Historically this value was stored in the executor, who's 
job it was to control/enforce
+        # it. However, with multiple executors, any of which can run up to 
core.parallelism TIs individually,
+        # we need to make sure in the scheduler now that we don't schedule 
more than core.parallelism totally
+        # across all executors.
+        num_occupied_slots = sum([executor.slots_occupied for executor in 
self.job.executors])
+        parallelism = conf.getint("core", "parallelism")
         if self.job.max_tis_per_query == 0:
-            max_tis = self.job.executor.slots_available
+            max_tis = parallelism - num_occupied_slots
         else:
-            max_tis = min(self.job.max_tis_per_query, 
self.job.executor.slots_available)
+            max_tis = min(self.job.max_tis_per_query, parallelism - 
num_occupied_slots)
+        if max_tis <= 0:
+            return 0
+
         queued_tis = self._executable_task_instances_to_queued(max_tis, 
session=session)
 
-        self._enqueue_task_instances_with_queued_state(queued_tis, 
session=session)
+        # Sort queued TIs to there respective executor
+        executor_to_queued_tis = self._executor_to_tis(queued_tis)
+        for executor, queued_tis_per_executor in 
executor_to_queued_tis.items():
+            self.log.info(
+                "Trying to enqueue tasks: %s for executor: %s",
+                queued_tis_per_executor,
+                executor,
+            )
+
+            
self._enqueue_task_instances_with_queued_state(queued_tis_per_executor, 
executor, session=session)
+
         return len(queued_tis)
 
-    def _process_executor_events(self, session: Session) -> int:
+    def _process_executor_events(self, executor: BaseExecutor, session: 
Session) -> int:
         """Respond to executor events."""
         if not self._standalone_dag_processor and not self.processor_agent:
             raise ValueError("Processor agent is not started.")
         ti_primary_key_to_try_number_map: dict[tuple[str, str, str, int], int] 
= {}
-        event_buffer = self.job.executor.get_event_buffer()
+        event_buffer = executor.get_event_buffer()
         tis_with_right_state: list[TaskInstanceKey] = []
 
         # Report execution
@@ -725,8 +786,8 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             msg = (
                 "TaskInstance Finished: dag_id=%s, task_id=%s, run_id=%s, 
map_index=%s, "
                 "run_start_date=%s, run_end_date=%s, "
-                "run_duration=%s, state=%s, executor_state=%s, try_number=%s, 
max_tries=%s, job_id=%s, "
-                "pool=%s, queue=%s, priority_weight=%d, operator=%s, 
queued_dttm=%s, "
+                "run_duration=%s, state=%s, executor=%s, executor_state=%s, 
try_number=%s, max_tries=%s, "
+                "job_id=%s, pool=%s, queue=%s, priority_weight=%d, 
operator=%s, queued_dttm=%s, "
                 "queued_by_job_id=%s, pid=%s"
             )
             self.log.info(
@@ -739,6 +800,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
                 ti.end_date,
                 ti.duration,
                 ti.state,
+                executor,
                 state,
                 try_number,
                 ti.max_tries,
@@ -765,7 +827,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             ti_queued = ti.try_number == buffer_key.try_number and ti.state == 
TaskInstanceState.QUEUED
             ti_requeued = (
                 ti.queued_by_job_id != self.job.id  # Another scheduler has 
queued this task again
-                or self.job.executor.has_task(ti)  # This scheduler has this 
task already
+                or executor.has_task(ti)  # This scheduler has this task 
already
             )
 
             if ti_queued and not ti_requeued:
@@ -774,9 +836,9 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
                     tags={"dag_id": ti.dag_id, "task_id": ti.task_id},
                 )
                 msg = (
-                    "The executor reported that the task instance %s finished 
with state %s, but the task instance's state attribute is %s. "  # noqa: 
RUF100, UP031, flynt
+                    "Executor %s reported that the task instance %s finished 
with state %s, but the task instance's state attribute is %s. "  # noqa: 
RUF100, UP031, flynt
                     "Learn more: 
https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally";
-                    % (ti, state, ti.state)
+                    % (executor, ti, state, ti.state)
                 )
                 if info is not None:
                     msg += " Extra info: %s" % info  # noqa: RUF100, UP031, 
flynt
@@ -798,7 +860,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
                         msg=msg,
                         processor_subdir=ti.dag_model.processor_subdir,
                     )
-                    self.job.executor.send_callback(request)
+                    executor.send_callback(request)
                 else:
                     ti.handle_failure(error=msg, session=session)
 
@@ -989,11 +1051,21 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
                     self.processor_agent.wait_until_finished()
 
                 with create_session() as session:
+                    # This will schedule for as many executors as possible.
                     num_queued_tis = self._do_scheduling(session)
 
-                    self.job.executor.heartbeat()
+                    # Heartbeat all executors, even if they're not receiving 
new tasks this loop. It will be
+                    # either a no-op, or they will check-in on currently 
running tasks and send out new
+                    # events to be processed below.
+                    for executor in self.job.executors:
+                        executor.heartbeat()
+
                     session.expunge_all()
-                    num_finished_events = 
self._process_executor_events(session=session)
+                    num_finished_events = 0
+                    for executor in self.job.executors:
+                        num_finished_events += self._process_executor_events(
+                            executor=executor, session=session
+                        )
                 if self.processor_agent:
                     self.processor_agent.heartbeat()
 
@@ -1092,16 +1164,18 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             session.expunge_all()
             # END: schedule TIs
 
-            if self.job.executor.slots_available <= 0:
+            # Attempt to schedule even if some executors are full but not all.
+            total_free_executor_slots = sum([executor.slots_available for 
executor in self.job.executors])
+            if total_free_executor_slots <= 0:
                 # We know we can't do anything here, so don't even try!
-                self.log.debug("Executor full, skipping critical section")
+                self.log.debug("All executors are full, skipping critical 
section")
                 num_queued_tis = 0
             else:
                 try:
                     timer = Stats.timer("scheduler.critical_section_duration")
                     timer.start()
 
-                    # Find anything TIs in state SCHEDULED, try to QUEUE it 
(send it to the executor)
+                    # Find any TIs in state SCHEDULED, try to QUEUE them (send 
it to the executors)
                     num_queued_tis = 
self._critical_section_enqueue_task_instances(session=session)
 
                     # Make sure we only sent this metric if we obtained the 
lock, otherwise we'll skew the
@@ -1841,12 +1915,32 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
     def _executor_to_tis(self, tis: list[TaskInstance]) -> dict[BaseExecutor, 
list[TaskInstance]]:
         """Organize TIs into lists per their respective executor."""
         _executor_to_tis: defaultdict[BaseExecutor, list[TaskInstance]] = 
defaultdict(list)
-        executor: str | None
         for ti in tis:
-            if ti.executor:
-                executor = str(ti.executor)
-            else:
-                executor = None
-            _executor_to_tis[ExecutorLoader.load_executor(executor)].append(ti)
+            if executor_obj := self._try_to_load_executor(ti.executor):
+                _executor_to_tis[executor_obj].append(ti)
 
         return _executor_to_tis
+
+    def _try_to_load_executor(self, executor_name: str | None) -> BaseExecutor 
| None:
+        """
+        Try to load the given executor.
+
+        In this context, we don't want to fail if the executor does not exist. 
Catch the exception and
+        log to the user.
+        """
+        try:
+            return ExecutorLoader.load_executor(executor_name)
+        except ValueError as e:
+            # This case should not happen unless some (as of now unknown) edge 
case occurs or direct DB
+            # modification, since the DAG parser will validate the tasks in 
the DAG and ensure the executor
+            # they request is available and if not, disallow the DAG to be 
scheduled.
+            # Keeping this exception handling because this is a critical issue 
if we do somehow find
+            # ourselves here and the user should get some feedback about that.
+            if "Unknown executor" in str(e):
+                self.log.warning(
+                    "Executor, %s, was not found but a Task was configured to 
use it", executor_name
+                )
+                return None
+            else:
+                # Re-raise any other Exception not related to unknown 
executors.
+                raise
diff --git a/tests/executors/test_base_executor.py 
b/tests/executors/test_base_executor.py
index 0e75751faf..e6dc057ca5 100644
--- a/tests/executors/test_base_executor.py
+++ b/tests/executors/test_base_executor.py
@@ -61,6 +61,11 @@ def test_infinite_slotspool():
     assert executor.slots_available == sys.maxsize
 
 
+def test_new_exec_no_slots_occupied():
+    executor = BaseExecutor(0)
+    assert executor.slots_occupied == 0
+
+
 def test_get_task_log():
     executor = BaseExecutor()
     ti = TaskInstance(task=BaseOperator(task_id="dummy"))
@@ -108,6 +113,7 @@ def test_fail_and_success():
     executor.success(key3, success_state)
 
     assert len(executor.running) == 0
+    assert executor.slots_occupied == 0
     assert len(executor.get_event_buffer()) == 3
 
 
@@ -168,9 +174,12 @@ def enqueue_tasks(executor, dagrun):
         executor.queue_command(task_instance, ["airflow"])
 
 
-def setup_trigger_tasks(dag_maker):
+def setup_trigger_tasks(dag_maker, parallelism=None):
     dagrun = setup_dagrun(dag_maker)
-    executor = BaseExecutor()
+    if parallelism:
+        executor = BaseExecutor(parallelism=parallelism)
+    else:
+        executor = BaseExecutor()
     executor.execute_async = mock.Mock()
     enqueue_tasks(executor, dagrun)
     return executor, dagrun
@@ -179,8 +188,21 @@ def setup_trigger_tasks(dag_maker):
 @pytest.mark.db_test
 @pytest.mark.parametrize("open_slots", [1, 2, 3])
 def test_trigger_queued_tasks(dag_maker, open_slots):
-    executor, _ = setup_trigger_tasks(dag_maker)
+    executor_parallelism = 10
+    executor, dagrun = setup_trigger_tasks(dag_maker, executor_parallelism)
+    num_tasks = len(dagrun.task_instances)
+
+    # All tasks are queued in setup method
+    assert executor.slots_occupied == num_tasks
+    assert executor.slots_available == executor_parallelism - num_tasks
+    assert len(executor.queued_tasks) == num_tasks
+    assert len(executor.running) == 0
     executor.trigger_tasks(open_slots)
+    assert executor.slots_available == executor_parallelism - num_tasks
+    assert executor.slots_occupied == num_tasks
+    assert len(executor.queued_tasks) == num_tasks - open_slots
+    # Only open_slots number of tasks are allowed through to running
+    assert len(executor.running) == open_slots
     assert executor.execute_async.call_count == open_slots
 
 
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index a282456c1f..3667810b4a 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -174,9 +174,53 @@ class TestSchedulerJob:
         second_executor.name = MagicMock(alias="secondary_exec", 
module_path="secondary.exec.module.path")
         with mock.patch("airflow.jobs.job.Job.executors", 
new_callable=PropertyMock) as executors_mock:
             with mock.patch("airflow.jobs.job.Job.executor", 
new_callable=PropertyMock) as executor_mock:
-                executor_mock.return_value = default_executor
-                executors_mock.return_value = [default_executor, 
second_executor]
-                yield [default_executor, second_executor]
+                with mock.patch(
+                    
"airflow.executors.executor_loader.ExecutorLoader.load_executor"
+                ) as loader_mock:
+                    # The executors are mocked, so cannot be loaded/imported. 
Mock load_executor and return
+                    # the correct object for the given input executor name.
+                    loader_mock.side_effect = lambda *x: {
+                        (None,): default_executor,
+                        ("default_exec",): default_executor,
+                        ("default.exec.module.path",): default_executor,
+                        ("secondary_exec",): second_executor,
+                        ("secondary.exec.module.path",): second_executor,
+                    }[x]
+
+                    from airflow.jobs import scheduler_job_runner
+
+                    # Reload the scheduler job runner so that it gets a fresh 
mocked copy of the exec loader
+                    reload(scheduler_job_runner)
+
+                    executor_mock.return_value = default_executor
+                    executors_mock.return_value = [default_executor, 
second_executor]
+                    yield [default_executor, second_executor]
+
+    @pytest.fixture
+    def mock_executor(self):
+        default_executor = mock.MagicMock(slots_available=8, slots_occupied=0)
+        default_executor.name = MagicMock(alias="default_exec", 
module_path="default.exec.module.path")
+        with mock.patch("airflow.jobs.job.Job.executors", 
new_callable=PropertyMock) as executors_mock:
+            with mock.patch("airflow.jobs.job.Job.executor", 
new_callable=PropertyMock) as executor_mock:
+                with mock.patch(
+                    
"airflow.executors.executor_loader.ExecutorLoader.load_executor"
+                ) as loader_mock:
+                    executor_mock.return_value = default_executor
+                    executors_mock.return_value = [default_executor]
+                    # The executor is mocked, so cannot be loaded/imported. 
Mock load_executor and return the
+                    # correct object for the given input executor name.
+                    loader_mock.side_effect = lambda *x: {
+                        ("default_exec",): default_executor,
+                        ("default.exec.module.path",): default_executor,
+                        (None,): default_executor,
+                    }[x]
+
+                    from airflow.jobs import scheduler_job_runner
+
+                    # Reload the scheduler job runner so that it gets a fresh 
mocked copy of the exec loader
+                    reload(scheduler_job_runner)
+
+                    yield default_executor
 
     @pytest.mark.parametrize(
         "configs",
@@ -274,7 +318,7 @@ class TestSchedulerJob:
 
         executor.event_buffer[ti1.key] = State.FAILED, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.FAILED
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -286,7 +330,7 @@ class TestSchedulerJob:
         session.commit()
         executor.event_buffer[ti1.key] = State.SUCCESS, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.SUCCESS
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -338,7 +382,7 @@ class TestSchedulerJob:
 
         executor.event_buffer[ti1.key] = State.FAILED, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.UP_FOR_RETRY
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -349,7 +393,7 @@ class TestSchedulerJob:
         session.commit()
         executor.event_buffer[ti1.key] = State.SUCCESS, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.SUCCESS
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -381,6 +425,7 @@ class TestSchedulerJob:
         task_callback = mock.MagicMock()
         mock_task_callback.return_value = task_callback
         scheduler_job = Job(executor=executor)
+        scheduler_job.executors = [executor]
         self.job_runner = SchedulerJobRunner(scheduler_job)
         self.job_runner.processor_agent = mock.MagicMock()
         session = settings.Session()
@@ -391,7 +436,7 @@ class TestSchedulerJob:
 
         executor.event_buffer[ti1.key] = State.FAILED, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db()
         # The state will remain in queued here and
         # will be set to failed in dag parsing process
@@ -400,7 +445,7 @@ class TestSchedulerJob:
             full_filepath=dag.fileloc,
             simple_task_instance=mock.ANY,
             processor_subdir=None,
-            msg="The executor reported that the task instance "
+            msg=f"Executor {executor} reported that the task instance "
             "<TaskInstance: 
test_process_executor_events_with_callback.dummy_task test [queued]> "
             "finished with state failed, but the task instance's state 
attribute is queued. "
             "Learn more: 
https://airflow.apache.org/docs/apache-airflow/stable/troubleshooting.html#task-state-changed-externally";,
@@ -442,7 +487,7 @@ class TestSchedulerJob:
         session.commit()
 
         executor.event_buffer[ti1.key] = State.FAILED, None
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db()
         assert ti1.state == State.FAILED
 
@@ -476,7 +521,7 @@ class TestSchedulerJob:
 
         executor.event_buffer[ti1.key.with_try_number(1)] = State.SUCCESS, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.QUEUED
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -489,7 +534,7 @@ class TestSchedulerJob:
 
         executor.event_buffer[ti1.key] = State.SUCCESS, None
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.QUEUED
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -503,7 +548,7 @@ class TestSchedulerJob:
         executor.event_buffer[ti1.key] = State.SUCCESS, None
         executor.has_task = mock.MagicMock(return_value=True)
 
-        self.job_runner._process_executor_events(session=session)
+        self.job_runner._process_executor_events(executor=executor, 
session=session)
         ti1.refresh_from_db(session=session)
         assert ti1.state == State.QUEUED
         scheduler_job.executor.callback_sink.send.assert_not_called()
@@ -567,16 +612,8 @@ class TestSchedulerJob:
 
         assert isinstance(scheduler_job.executor.callback_sink, 
PipeCallbackSink)
 
-    
@mock.patch("airflow.executors.executor_loader.ExecutorLoader.init_executors")
-    
@mock.patch("airflow.executors.executor_loader.ExecutorLoader.get_default_executor")
     @conf_vars({("scheduler", "standalone_dag_processor"): "False"})
-    def 
test_setup_callback_sink_not_standalone_dag_processor_multiple_executors(
-        self, get_default_executor_mock, init_executors_mock
-    ):
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        init_executors_mock.return_value = [default_executor, second_executor]
-        get_default_executor_mock.return_value = default_executor
+    def 
test_setup_callback_sink_not_standalone_dag_processor_multiple_executors(self, 
mock_executors):
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         self.job_runner._execute()
@@ -592,16 +629,8 @@ class TestSchedulerJob:
 
         assert isinstance(scheduler_job.executor.callback_sink, 
DatabaseCallbackSink)
 
-    @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
-    @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
     @conf_vars({("scheduler", "standalone_dag_processor"): "True"})
-    def test_setup_callback_sink_standalone_dag_processor_multiple_executors(
-        self, executor_mock, executors_mock
-    ):
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        executor_mock.return_value = default_executor
-        executors_mock.return_value = [default_executor, second_executor]
+    def 
test_setup_callback_sink_standalone_dag_processor_multiple_executors(self, 
mock_executors):
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         self.job_runner._execute()
@@ -609,14 +638,8 @@ class TestSchedulerJob:
         for executor in scheduler_job.executors:
             assert isinstance(executor.callback_sink, DatabaseCallbackSink)
 
-    @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
-    @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
     @conf_vars({("scheduler", "standalone_dag_processor"): "True"})
-    def test_executor_start_called(self, executor_mock, executors_mock):
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        executor_mock.return_value = default_executor
-        executors_mock.return_value = [default_executor, second_executor]
+    def test_executor_start_called(self, mock_executors):
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         self.job_runner._execute()
@@ -625,13 +648,7 @@ class TestSchedulerJob:
         for executor in scheduler_job.executors:
             executor.start.assert_called_once()
 
-    @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
-    @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
-    def test_executor_job_id_assigned(self, executor_mock, executors_mock):
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        executor_mock.return_value = default_executor
-        executors_mock.return_value = [default_executor, second_executor]
+    def test_executor_job_id_assigned(self, mock_executors):
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         self.job_runner._execute()
@@ -640,13 +657,23 @@ class TestSchedulerJob:
         for executor in scheduler_job.executors:
             assert executor.job_id == scheduler_job.id
 
-    @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
-    @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
-    def test_executor_debug_dump(self, executor_mock, executors_mock):
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        executor_mock.return_value = default_executor
-        executors_mock.return_value = [default_executor, second_executor]
+    def test_executor_heartbeat(self, mock_executors):
+        scheduler_job = Job()
+        self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
+        self.job_runner._execute()
+
+        for executor in scheduler_job.executors:
+            executor.heartbeat.assert_called_once()
+
+    def test_executor_events_processed(self, mock_executors):
+        scheduler_job = Job()
+        self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
+        self.job_runner._execute()
+
+        for executor in scheduler_job.executors:
+            executor.get_event_buffer.assert_called_once()
+
+    def test_executor_debug_dump(self, mock_executors):
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         self.job_runner._debug_dump(1, mock.MagicMock())
@@ -824,6 +851,45 @@ class TestSchedulerJob:
         assert [ti.key for ti in res] == [tis[1].key]
         session.rollback()
 
+    def test_find_executable_task_instances_executor(self, dag_maker, 
mock_executors):
+        """
+        Test that tasks for all executors are set to queued, if space allows it
+        """
+
+        scheduler_job = Job()
+        self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
+        session = settings.Session()
+
+        dag_id = 
"SchedulerJobTest.test_find_executable_task_instances_executor"
+
+        with dag_maker(dag_id=dag_id):
+            op1 = EmptyOperator(task_id="dummy1")  # No executor specified, 
runs on default executor
+            op2 = EmptyOperator(task_id="dummy2", executor="default_exec")
+            op3 = EmptyOperator(task_id="dummy3", 
executor="default.exec.module.path")
+            op4 = EmptyOperator(task_id="dummy4", executor="secondary_exec")
+            op5 = EmptyOperator(task_id="dummy5", 
executor="secondary.exec.module.path")
+
+        dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
+
+        ti1 = dag_run.get_task_instance(op1.task_id, session)
+        ti2 = dag_run.get_task_instance(op2.task_id, session)
+        ti3 = dag_run.get_task_instance(op3.task_id, session)
+        ti4 = dag_run.get_task_instance(op4.task_id, session)
+        ti5 = dag_run.get_task_instance(op5.task_id, session)
+
+        tis_tuple = (ti1, ti2, ti3, ti4, ti5)
+        for ti in tis_tuple:
+            ti.state = State.SCHEDULED
+
+        session.flush()
+
+        res = self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
+
+        assert 5 == len(res)
+        res_ti_keys = [res_ti.key for res_ti in res]
+        for ti in tis_tuple:
+            assert ti.key in res_ti_keys
+
     def test_find_executable_task_instances_order_priority_with_pools(self, 
dag_maker):
         """
         The scheduler job should pick tasks with higher priority for execution
@@ -892,14 +958,14 @@ class TestSchedulerJob:
         assert [ti.key for ti in res] == [tis[1].key]
         session.rollback()
 
-    def test_find_executable_task_instances_in_default_pool(self, dag_maker):
+    def test_find_executable_task_instances_in_default_pool(self, dag_maker, 
mock_executor):
         set_default_pool_slots(1)
 
         dag_id = 
"SchedulerJobTest.test_find_executable_task_instances_in_default_pool"
         with dag_maker(dag_id=dag_id):
             op1 = EmptyOperator(task_id="dummy1")
             op2 = EmptyOperator(task_id="dummy2")
-        scheduler_job = Job(executor=MockExecutor())
+        scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
 
         session = settings.Session()
@@ -1193,58 +1259,60 @@ class TestSchedulerJob:
         session.merge(ti2)
         session.flush()
 
-        res = self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
-
-        assert 2 == len(res)
-
-        ti1_1.state = State.RUNNING
-        ti2.state = State.RUNNING
-        ti1_2 = dr2.get_task_instance(task1.task_id)
-        ti1_2.state = State.SCHEDULED
-        session.merge(ti1_1)
-        session.merge(ti2)
-        session.merge(ti1_2)
-        session.flush()
+        with 
mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as 
loader_mock:
+            loader_mock.side_effect = executor.get_mock_loader_side_effect()
+            res = 
self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
+
+            assert 2 == len(res)
+
+            ti1_1.state = State.RUNNING
+            ti2.state = State.RUNNING
+            ti1_2 = dr2.get_task_instance(task1.task_id)
+            ti1_2.state = State.SCHEDULED
+            session.merge(ti1_1)
+            session.merge(ti2)
+            session.merge(ti1_2)
+            session.flush()
 
-        res = self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
+            res = 
self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
 
-        assert 1 == len(res)
+            assert 1 == len(res)
 
-        ti1_2.state = State.RUNNING
-        ti1_3 = dr3.get_task_instance(task1.task_id)
-        ti1_3.state = State.SCHEDULED
-        session.merge(ti1_2)
-        session.merge(ti1_3)
-        session.flush()
+            ti1_2.state = State.RUNNING
+            ti1_3 = dr3.get_task_instance(task1.task_id)
+            ti1_3.state = State.SCHEDULED
+            session.merge(ti1_2)
+            session.merge(ti1_3)
+            session.flush()
 
-        res = self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
+            res = 
self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
 
-        assert 0 == len(res)
+            assert 0 == len(res)
 
-        ti1_1.state = State.SCHEDULED
-        ti1_2.state = State.SCHEDULED
-        ti1_3.state = State.SCHEDULED
-        session.merge(ti1_1)
-        session.merge(ti1_2)
-        session.merge(ti1_3)
-        session.flush()
+            ti1_1.state = State.SCHEDULED
+            ti1_2.state = State.SCHEDULED
+            ti1_3.state = State.SCHEDULED
+            session.merge(ti1_1)
+            session.merge(ti1_2)
+            session.merge(ti1_3)
+            session.flush()
 
-        res = self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
+            res = 
self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
 
-        assert 2 == len(res)
+            assert 2 == len(res)
 
-        ti1_1.state = State.RUNNING
-        ti1_2.state = State.SCHEDULED
-        ti1_3.state = State.SCHEDULED
-        session.merge(ti1_1)
-        session.merge(ti1_2)
-        session.merge(ti1_3)
-        session.flush()
+            ti1_1.state = State.RUNNING
+            ti1_2.state = State.SCHEDULED
+            ti1_3.state = State.SCHEDULED
+            session.merge(ti1_1)
+            session.merge(ti1_2)
+            session.merge(ti1_3)
+            session.flush()
 
-        res = self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
+            res = 
self.job_runner._executable_task_instances_to_queued(max_tis=32, 
session=session)
 
-        assert 1 == len(res)
-        session.rollback()
+            assert 1 == len(res)
+            session.rollback()
 
     def 
test_change_state_for_executable_task_instances_no_tis_with_state(self, 
dag_maker):
         dag_id = "SchedulerJobTest.test_change_state_for__no_tis_with_state"
@@ -1533,7 +1601,9 @@ class TestSchedulerJob:
         ti1 = dr1.get_task_instance(task1.task_id, session)
 
         with patch.object(BaseExecutor, "queue_command") as mock_queue_command:
-            self.job_runner._enqueue_task_instances_with_queued_state([ti1], 
session=session)
+            self.job_runner._enqueue_task_instances_with_queued_state(
+                [ti1], executor=scheduler_job.executor, session=session
+            )
 
         assert mock_queue_command.called
         session.rollback()
@@ -1557,13 +1627,23 @@ class TestSchedulerJob:
         session.commit()
 
         with patch.object(BaseExecutor, "queue_command") as mock_queue_command:
-            self.job_runner._enqueue_task_instances_with_queued_state([ti], 
session=session)
+            self.job_runner._enqueue_task_instances_with_queued_state(
+                [ti], executor=scheduler_job.executor, session=session
+            )
         session.flush()
         ti.refresh_from_db(session=session)
         assert ti.state == State.NONE
         mock_queue_command.assert_not_called()
 
-    def test_critical_section_enqueue_task_instances(self, dag_maker):
+    @pytest.mark.parametrize(
+        "task1_exec, task2_exec",
+        [
+            ("default_exec", "default_exec"),
+            ("default_exec", "secondary_exec"),
+            ("secondary_exec", "secondary_exec"),
+        ],
+    )
+    def test_critical_section_enqueue_task_instances(self, task1_exec, 
task2_exec, dag_maker, mock_executors):
         dag_id = "SchedulerJobTest.test_execute_task_instances"
         task_id_1 = "dummy_task"
         task_id_2 = "dummy_task_nonexistent_queue"
@@ -1573,13 +1653,13 @@ class TestSchedulerJob:
         # check the num tasks once so if max_active_tasks was 3,
         # we could execute arbitrarily many tasks in the second run
         with dag_maker(dag_id=dag_id, max_active_tasks=3, session=session) as 
dag:
-            task1 = EmptyOperator(task_id=task_id_1)
-            task2 = EmptyOperator(task_id=task_id_2)
+            task1 = EmptyOperator(task_id=task_id_1, executor=task1_exec)
+            task2 = EmptyOperator(task_id=task_id_2, executor=task2_exec)
 
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
 
-        # create first dag run with 1 running and 1 queued
+        # create first dag run with 2 running tasks
 
         dr1 = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED)
 
@@ -1620,7 +1700,10 @@ class TestSchedulerJob:
         assert {State.QUEUED, State.SCHEDULED} == {ti3.state, ti4.state}
         assert 1 == res
 
-    def test_execute_task_instances_limit(self, dag_maker):
+        res = self.job_runner._critical_section_enqueue_task_instances(session)
+        assert 0 == res
+
+    def test_execute_task_instances_limit_second_executor(self, dag_maker, 
mock_executors):
         dag_id = "SchedulerJobTest.test_execute_task_instances_limit"
         task_id_1 = "dummy_task"
         task_id_2 = "dummy_task_2"
@@ -1630,8 +1713,8 @@ class TestSchedulerJob:
         # check the num tasks once so if max_active_tasks was 3,
         # we could execute arbitrarily many tasks in the second run
         with dag_maker(dag_id=dag_id, max_active_tasks=16, session=session):
-            task1 = EmptyOperator(task_id=task_id_1)
-            task2 = EmptyOperator(task_id=task_id_2)
+            task1 = EmptyOperator(task_id=task_id_1, executor="default_exec")
+            task2 = EmptyOperator(task_id=task_id_2, executor="secondary_exec")
 
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
@@ -1647,33 +1730,135 @@ class TestSchedulerJob:
                 )
                 yield dagrun
 
-        tis = []
+        tis1 = []
+        tis2 = []
         for dr in _create_dagruns():
             ti1 = dr.get_task_instance(task1.task_id, session)
+            tis1.append(ti1)
             ti2 = dr.get_task_instance(task2.task_id, session)
+            tis2.append(ti2)
             ti1.state = State.SCHEDULED
             ti2.state = State.SCHEDULED
             session.flush()
-        scheduler_job.max_tis_per_query = 2
+        scheduler_job.max_tis_per_query = 6
+        # First pass we'll grab 6 of the 8 tasks (limited by max_tis_per_query)
+        res = self.job_runner._critical_section_enqueue_task_instances(session)
+        assert 6 == res
+        for ti in tis1[:3] + tis2[:3]:
+            ti.refresh_from_db()
+            assert ti.state == TaskInstanceState.QUEUED
+        for ti in tis1[3:] + tis2[3:]:
+            ti.refresh_from_db()
+            assert ti.state == TaskInstanceState.SCHEDULED
+
+        # The remaining TIs are queued
         res = self.job_runner._critical_section_enqueue_task_instances(session)
         assert 2 == res
 
+        for ti in tis1 + tis2:
+            ti.refresh_from_db()
+            assert State.QUEUED == ti.state
+
+    @pytest.mark.parametrize(
+        "task1_exec, task2_exec",
+        [
+            ("default_exec", "default_exec"),
+            ("default_exec", "secondary_exec"),
+            ("secondary_exec", "secondary_exec"),
+        ],
+    )
+    def test_execute_task_instances_limit(self, task1_exec, task2_exec, 
dag_maker, mock_executors):
+        dag_id = "SchedulerJobTest.test_execute_task_instances_limit"
+        task_id_1 = "dummy_task"
+        task_id_2 = "dummy_task_2"
+        session = settings.Session()
+        # important that len(tasks) is less than max_active_tasks
+        # because before scheduler._execute_task_instances would only
+        # check the num tasks once so if max_active_tasks was 3,
+        # we could execute arbitrarily many tasks in the second run
+        with dag_maker(dag_id=dag_id, max_active_tasks=16, session=session):
+            task1 = EmptyOperator(task_id=task_id_1, executor=task1_exec)
+            task2 = EmptyOperator(task_id=task_id_2, executor=task2_exec)
+
+        scheduler_job = Job()
+        self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
+
+        def _create_dagruns():
+            dagrun = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, 
state=State.RUNNING)
+            yield dagrun
+            for _ in range(3):
+                dagrun = dag_maker.create_dagrun_after(
+                    dagrun,
+                    run_type=DagRunType.SCHEDULED,
+                    state=State.RUNNING,
+                )
+                yield dagrun
+
+        tis = []
+        for dr in _create_dagruns():
+            ti1 = dr.get_task_instance(task1.task_id, session)
+            tis.append(ti1)
+            ti2 = dr.get_task_instance(task2.task_id, session)
+            tis.append(ti2)
+            ti1.state = State.SCHEDULED
+            ti2.state = State.SCHEDULED
+            session.flush()
+        scheduler_job.max_tis_per_query = 2
+
+        total_enqueued = 
self.job_runner._critical_section_enqueue_task_instances(session)
+        assert 2 == total_enqueued
+
+    def test_execute_task_instances_limit_slots(self, dag_maker, 
mock_executors):
+        dag_id = "SchedulerJobTest.test_execute_task_instances_limit"
+        task_id_1 = "dummy_task"
+        task_id_2 = "dummy_task_2"
+        session = settings.Session()
+        # important that len(tasks) is less than max_active_tasks
+        # because before scheduler._execute_task_instances would only
+        # check the num tasks once so if max_active_tasks was 3,
+        # we could execute arbitrarily many tasks in the second run
+        with dag_maker(dag_id=dag_id, max_active_tasks=16, session=session):
+            task1 = EmptyOperator(task_id=task_id_1, executor="default_exec")
+            task2 = EmptyOperator(task_id=task_id_2, executor="secondary_exec")
+
+        scheduler_job = Job()
+        self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
+
+        def _create_dagruns():
+            dagrun = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, 
state=State.RUNNING)
+            yield dagrun
+            for _ in range(3):
+                dagrun = dag_maker.create_dagrun_after(
+                    dagrun,
+                    run_type=DagRunType.SCHEDULED,
+                    state=State.RUNNING,
+                )
+                yield dagrun
+
+        tis = []
+        for dr in _create_dagruns():
+            ti1 = dr.get_task_instance(task1.task_id, session)
+            tis.append(ti1)
+            ti2 = dr.get_task_instance(task2.task_id, session)
+            tis.append(ti2)
+            ti1.state = State.SCHEDULED
+            ti2.state = State.SCHEDULED
+            session.flush()
+
         scheduler_job.max_tis_per_query = 8
-        with mock.patch.object(
-            type(scheduler_job.executor), "slots_available", 
new_callable=mock.PropertyMock
-        ) as mock_slots:
-            mock_slots.return_value = 2
-            # Check that we don't "overfill" the executor
-            assert 2 == res
-            res = 
self.job_runner._critical_section_enqueue_task_instances(session)
+        scheduler_job.executor.slots_available = 2  # Limit only the default 
executor to 2 slots.
+        # Check that we don't "overfill" an executor when the max tis per 
query is larger than slots
+        # available. Of the 8 tasks returned by the query, the default 
executor will only take 2 and the
+        # secondary executor will take 4 (since only 4 of the 8 TIs in the 
result will be for that executor)
+        res = self.job_runner._critical_section_enqueue_task_instances(session)
+        assert res == 6
 
+        scheduler_job.executor.slots_available = 6  # The default executor has 
more slots freed now and
+        # will take the other two TIs.
         res = self.job_runner._critical_section_enqueue_task_instances(session)
-        assert 4 == res
-        for ti in tis:
-            ti.refresh_from_db()
-            assert State.QUEUED == ti.state
+        assert res == 2
 
-    def test_execute_task_instances_unlimited(self, dag_maker):
+    def test_execute_task_instances_unlimited(self, dag_maker, mock_executor):
         """Test that max_tis_per_query=0 is unlimited"""
 
         dag_id = "SchedulerJobTest.test_execute_task_instances_unlimited"
@@ -1706,11 +1891,73 @@ class TestSchedulerJob:
             ti2.state = State.SCHEDULED
             session.flush()
         scheduler_job.max_tis_per_query = 0
-        scheduler_job.executor = MagicMock(slots_available=36)
+        scheduler_job.executor.parallelism = 32
+        scheduler_job.executor.slots_available = 31
 
         res = self.job_runner._critical_section_enqueue_task_instances(session)
         # 20 dag runs * 2 tasks each = 40, but limited by number of slots 
available
-        assert res == 36
+        assert res == 31
+        session.rollback()
+
+    @pytest.mark.parametrize(
+        "task1_exec, task2_exec",
+        [
+            ("default_exec", "default_exec"),
+            ("default_exec", "secondary_exec"),
+            ("secondary_exec", "secondary_exec"),
+        ],
+    )
+    def test_execute_task_instances_unlimited_multiple_executors(
+        self, task1_exec, task2_exec, dag_maker, mock_executors
+    ):
+        """Test that max_tis_per_query=0 is unlimited"""
+
+        dag_id = "SchedulerJobTest.test_execute_task_instances_unlimited"
+        task_id_1 = "dummy_task"
+        task_id_2 = "dummy_task_2"
+        session = settings.Session()
+
+        with dag_maker(dag_id=dag_id, max_active_tasks=1024, session=session):
+            task1 = EmptyOperator(task_id=task_id_1, executor=task1_exec)
+            task2 = EmptyOperator(task_id=task_id_2, executor=task2_exec)
+
+        scheduler_job = Job()
+        self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
+
+        def _create_dagruns():
+            dagrun = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, 
state=State.RUNNING)
+            yield dagrun
+            for _ in range(40):
+                dagrun = dag_maker.create_dagrun_after(
+                    dagrun,
+                    run_type=DagRunType.SCHEDULED,
+                    state=State.RUNNING,
+                )
+                yield dagrun
+
+        for dr in _create_dagruns():
+            ti1 = dr.get_task_instance(task1.task_id, session)
+            ti2 = dr.get_task_instance(task2.task_id, session)
+            ti1.state = State.SCHEDULED
+            ti2.state = State.SCHEDULED
+            session.flush()
+        scheduler_job.max_tis_per_query = 0
+        for executor in mock_executors:
+            executor.parallelism = 32
+            executor.slots_available = 31
+
+        total_enqueued = 0
+        with conf_vars({("core", "parallelism"): "40"}):
+            # 40 dag runs * 2 tasks each = 80. Two executors have capacity for 
61 concurrent jobs, but they
+            # together respect core.parallelism and will not run more in 
aggregate then that allows.
+            total_enqueued += 
self.job_runner._critical_section_enqueue_task_instances(session)
+
+        if task1_exec != task2_exec:
+            # Two executors will execute up to core parallelism
+            assert total_enqueued == 40
+        else:
+            # A single executor will only run up to its available slots
+            assert total_enqueued == 31
         session.rollback()
 
     def test_adopt_or_reset_orphaned_tasks(self, dag_maker):
@@ -1864,33 +2111,22 @@ class TestSchedulerJob:
         assert "Executor doesn't support cleanup of stuck queued tasks. 
Skipping." in caplog.text
 
     @mock.patch("airflow.dag_processing.manager.DagFileProcessorAgent")
-    def test_executor_end_called(self, mock_processor_agent):
+    def test_executor_end_called(self, mock_processor_agent, mock_executors):
         """
         Test to make sure executor.end gets called with a successful scheduler 
loop run
         """
-        scheduler_job = Job(executor=mock.MagicMock(slots_available=8))
+        scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         run_job(scheduler_job, execute_callable=self.job_runner._execute)
         scheduler_job.executor.end.assert_called_once()
         self.job_runner.processor_agent.end.assert_called_once()
 
-    @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
-    @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
     @mock.patch("airflow.dag_processing.manager.DagFileProcessorAgent")
-    def test_executor_end_called_multiple_executors(
-        self, mock_processor_agent, executor_mock, executors_mock
-    ):
+    def test_executor_end_called_multiple_executors(self, 
mock_processor_agent, mock_executors):
         """
         Test to make sure executor.end gets called on all executors with a 
successful scheduler loop run
         """
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        executor_mock.return_value = default_executor
-        executors_mock.return_value = [default_executor, second_executor]
         scheduler_job = Job()
-        assert scheduler_job.executor is default_executor
-        assert scheduler_job.executors == [default_executor, second_executor]
-
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         run_job(scheduler_job, execute_callable=self.job_runner._execute)
         scheduler_job.executor.end.assert_called_once()
@@ -1917,22 +2153,13 @@ class TestSchedulerJob:
         scheduler_job.executor.end.assert_called_once()
         mock_processor_agent.return_value.end.reset_mock(side_effect=True)
 
-    @mock.patch("airflow.jobs.job.Job.executors", new_callable=PropertyMock)
-    @mock.patch("airflow.jobs.job.Job.executor", new_callable=PropertyMock)
     @mock.patch("airflow.dag_processing.manager.DagFileProcessorAgent")
-    def test_cleanup_methods_all_called_multiple_executors(
-        self, mock_processor_agent, executor_mock, executors_mock
-    ):
+    def test_cleanup_methods_all_called_multiple_executors(self, 
mock_processor_agent, mock_executors):
         """
         Test to make sure all cleanup methods are called when the scheduler 
loop has an exception
         """
-        default_executor = mock.MagicMock(slots_available=8)
-        second_executor = mock.MagicMock(slots_available=8)
-        executor_mock.return_value = default_executor
-        executors_mock.return_value = [default_executor, second_executor]
         scheduler_job = Job()
-        assert scheduler_job.executor is default_executor
-        assert scheduler_job.executors == [default_executor, second_executor]
+
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull, num_runs=1)
         self.job_runner._run_scheduler_loop = 
mock.MagicMock(side_effect=RuntimeError("oops"))
         mock_processor_agent.return_value.end.side_effect = 
RuntimeError("double oops")
@@ -2604,7 +2831,7 @@ class TestSchedulerJob:
         dr.update_state()
         assert dr.state == State.RUNNING
 
-    def test_dagrun_root_after_dagrun_unfinished(self):
+    def test_dagrun_root_after_dagrun_unfinished(self, mock_executor):
         """
         DagRuns with one successful and one future root task -> SUCCESS
 
@@ -2615,7 +2842,7 @@ class TestSchedulerJob:
         dag = self.dagbag.get_dag(dag_id)
         dag.sync_to_db()
 
-        scheduler_job = Job(executor=self.null_exec)
+        scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, num_runs=1, 
subdir=dag.fileloc)
         run_job(scheduler_job, execute_callable=self.job_runner._execute)
 
@@ -2809,7 +3036,7 @@ class TestSchedulerJob:
             {("scheduler", "standalone_dag_processor"): "True"},
         ],
     )
-    def test_scheduler_verify_pool_full(self, dag_maker, configs):
+    def test_scheduler_verify_pool_full(self, dag_maker, configs, 
mock_executor):
         """
         Test task instances not queued when pool is full
         """
@@ -2826,7 +3053,7 @@ class TestSchedulerJob:
             session.add(pool)
             session.flush()
 
-            scheduler_job = Job(executor=self.null_exec)
+            scheduler_job = Job()
             self.job_runner = SchedulerJobRunner(job=scheduler_job)
             self.job_runner.processor_agent = mock.MagicMock()
 
@@ -2845,7 +3072,7 @@ class TestSchedulerJob:
             assert len(task_instances_list) == 1
 
     @pytest.mark.need_serialized_dag
-    def test_scheduler_verify_pool_full_2_slots_per_task(self, dag_maker, 
session):
+    def test_scheduler_verify_pool_full_2_slots_per_task(self, dag_maker, 
session, mock_executor):
         """
         Test task instances not queued when pool is full.
 
@@ -2867,7 +3094,7 @@ class TestSchedulerJob:
         session.add(pool)
         session.flush()
 
-        scheduler_job = Job(executor=self.null_exec)
+        scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job)
 
         self.job_runner.processor_agent = mock.MagicMock()
@@ -2890,7 +3117,7 @@ class TestSchedulerJob:
         # As tasks require 2 slots, only 3 can fit into 6 available
         assert len(task_instances_list) == 3
 
-    def test_scheduler_keeps_scheduling_pool_full(self, dag_maker):
+    def test_scheduler_keeps_scheduling_pool_full(self, dag_maker, 
mock_executor):
         """
         Test task instances in a pool that isn't full keep getting scheduled 
even when a pool is full.
         """
@@ -2923,7 +3150,7 @@ class TestSchedulerJob:
         session.add(pool_p2)
         session.flush()
 
-        scheduler_job = Job(executor=self.null_exec)
+        scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job)
         self.job_runner.processor_agent = mock.MagicMock()
 
@@ -2961,7 +3188,7 @@ class TestSchedulerJob:
             for task_instance in task_instances_list2
         )
 
-    def test_scheduler_verify_priority_and_slots(self, dag_maker):
+    def test_scheduler_verify_priority_and_slots(self, dag_maker, 
mock_executor):
         """
         Test task instances with higher priority are not queued
         when pool does not have enough slots.
@@ -2999,7 +3226,7 @@ class TestSchedulerJob:
         session.add(pool)
         session.flush()
 
-        scheduler_job = Job(executor=self.null_exec)
+        scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job)
 
         self.job_runner.processor_agent = mock.MagicMock()
@@ -3239,7 +3466,11 @@ class TestSchedulerJob:
             # Since the DAG is not in the directory watched by scheduler job,
             # it would've been marked as deleted and not being scheduled.
             with mock.patch.object(DagModel, "deactivate_deleted_dags"):
-                run_job(scheduler_job, 
execute_callable=self.job_runner._execute)
+                with mock.patch(
+                    
"airflow.executors.executor_loader.ExecutorLoader.load_executor"
+                ) as loader_mock:
+                    loader_mock.side_effect = 
executor.get_mock_loader_side_effect()
+                    run_job(scheduler_job, 
execute_callable=self.job_runner._execute)
 
         do_schedule()
         with create_session() as session:
@@ -4221,10 +4452,16 @@ class TestSchedulerJob:
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
 
-        scheduler_job.executor = MockExecutor(do_update=False)
-        self.job_runner.processor_agent = 
mock.MagicMock(spec=DagFileProcessorAgent)
+        executor = MockExecutor(do_update=False)
+        scheduler_job.executor = executor
+        scheduler_job.executors = [executor]
+        with 
mock.patch("airflow.executors.executor_loader.ExecutorLoader.load_executor") as 
loader_mock:
+            # The executor is mocked, so cannot be loaded/imported. Mock 
load_executor and return the
+            # correct object for the given input executor name.
+            loader_mock.side_effect = executor.get_mock_loader_side_effect()
+            self.job_runner.processor_agent = 
mock.MagicMock(spec=DagFileProcessorAgent)
 
-        num_queued = self.job_runner._do_scheduling(session)
+            num_queued = self.job_runner._do_scheduling(session)
         assert num_queued == 1
 
         session.flush()
@@ -4339,7 +4576,7 @@ class TestSchedulerJob:
         ]
         assert dagrun_execution_dates == expected_execution_dates
 
-    def test_do_schedule_max_active_runs_and_manual_trigger(self, dag_maker):
+    def test_do_schedule_max_active_runs_and_manual_trigger(self, dag_maker, 
mock_executors):
         """
         Make sure that when a DAG is already at max_active_runs, that manually 
triggered
         dagruns don't start running.
@@ -4369,7 +4606,6 @@ class TestSchedulerJob:
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job, 
subdir=os.devnull)
 
-        scheduler_job.executor = MockExecutor(do_update=False)
         self.job_runner.processor_agent = 
mock.MagicMock(spec=DagFileProcessorAgent)
 
         num_queued = self.job_runner._do_scheduling(session)
diff --git a/tests/test_utils/mock_executor.py 
b/tests/test_utils/mock_executor.py
index a02af122bf..5080290500 100644
--- a/tests/test_utils/mock_executor.py
+++ b/tests/test_utils/mock_executor.py
@@ -21,6 +21,7 @@ from collections import defaultdict
 from unittest.mock import MagicMock, Mock
 
 from airflow.executors.base_executor import BaseExecutor
+from airflow.executors.executor_utils import ExecutorName
 from airflow.models.taskinstancekey import TaskInstanceKey
 from airflow.utils.session import create_session
 from airflow.utils.state import State
@@ -32,6 +33,8 @@ class MockExecutor(BaseExecutor):
     """
 
     supports_pickling = False
+    mock_module_path = "mock.executor.path"
+    mock_alias = "mock_executor"
 
     def __init__(self, do_update=True, *args, **kwargs):
         self.do_update = do_update
@@ -43,6 +46,8 @@ class MockExecutor(BaseExecutor):
         # All the tasks, in a stable sort order
         self.sorted_tasks = []
 
+        self.name = ExecutorName(module_path=self.mock_module_path, 
alias=self.mock_alias)
+
         # If multiprocessing runs in spawn mode,
         # arguments are to be pickled but lambda is not picclable.
         # So we should pass self.success instead of lambda.
@@ -101,3 +106,10 @@ class MockExecutor(BaseExecutor):
         """
         assert isinstance(run_id, str)
         self.mock_task_results[TaskInstanceKey(dag_id, task_id, run_id, 
try_number)] = State.FAILED
+
+    def get_mock_loader_side_effect(self):
+        return lambda *x: {
+            (None,): self,
+            (self.mock_module_path,): self,
+            (self.mock_alias,): self,
+        }[x]

Reply via email to