This is an automated email from the ASF dual-hosted git repository.

dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 2035dc71be Support deferrable operators in dag test (#34585)
2035dc71be is described below

commit 2035dc71be68b6d8dcf4536370c6e21ee0fd9585
Author: Daniel Standish <[email protected]>
AuthorDate: Mon Sep 25 09:27:03 2023 -0700

    Support deferrable operators in dag test (#34585)
    
    You need to run triggerer separately, but now at least dag test will resume 
the task after deferral.
---
 airflow/models/dag.py          | 16 ++++++++++++++--
 airflow/models/taskinstance.py |  3 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index 4baf2dbf32..da734152c8 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -27,6 +27,7 @@ import os
 import pathlib
 import pickle
 import sys
+import time
 import traceback
 import warnings
 import weakref
@@ -123,7 +124,7 @@ from airflow.utils.sqlalchemy import (
     tuple_in_condition,
     with_row_locks,
 )
-from airflow.utils.state import DagRunState, TaskInstanceState
+from airflow.utils.state import DagRunState, State, TaskInstanceState
 from airflow.utils.trigger_rule import TriggerRule
 from airflow.utils.types import NOTSET, ArgNotSet, DagRunType, EdgeInfoType
 
@@ -2811,8 +2812,19 @@ class DAG(LoggingMixin):
         # for task readiness and dependency management. This is notably faster
         # than creating a BackfillJob and allows us to surface logs to the user
         while dr.state == DagRunState.RUNNING:
+            session.expire_all()
             schedulable_tis, _ = dr.update_state(session=session)
-            for ti in schedulable_tis:
+            for s in schedulable_tis:
+                s.state = TaskInstanceState.SCHEDULED
+            session.commit()
+            # triggerer may mark tasks scheduled so we read from DB
+            all_tis = set(dr.get_task_instances(session=session))
+            scheduled_tis = {x for x in all_tis if x.state == 
TaskInstanceState.SCHEDULED}
+            ids_unrunnable = {x for x in all_tis if x.state not in 
State.finished} - scheduled_tis
+            if not scheduled_tis and ids_unrunnable:
+                self.log.warning("No tasks to run. unrunnable tasks: %s", 
ids_unrunnable)
+                time.sleep(1)
+            for ti in scheduled_tis:
                 try:
                     add_logger_if_needed(ti)
                     ti.task = tasks[ti.task_id]
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index 98da686184..039fe68f98 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -1361,6 +1361,9 @@ class TaskInstance(Base, LoggingMixin):
         # can be changed when calling 'run'
         self.test_mode = False
 
+    def __hash__(self):
+        return hash((self.task_id, self.dag_id, self.run_id, self.map_index))
+
     @property
     def stats_tags(self) -> dict[str, str]:
         """Returns task instance tags."""

Reply via email to