This is an automated email from the ASF dual-hosted git repository.

ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 848a3969e6 Classic TaskGroup setup/teardown (#29891)
848a3969e6 is described below

commit 848a3969e65681ea7581a63f9dee3aba52dd316b
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Tue Mar 14 07:56:05 2023 +0100

    Classic TaskGroup setup/teardown (#29891)
    
    * Classic TaskGroup setup/teardown
    
    This implements classic TaskGroup setup/teardown. Ensures that nested
    TaskGroups are taken into account
    
    * fixup! Classic TaskGroup setup/teardown
---
 airflow/decorators/task_group.py        |   2 +
 airflow/utils/task_group.py             |  33 +++-
 tests/decorators/test_setup_teardown.py | 314 +++++++++++++++++++++++++++++++-
 3 files changed, 343 insertions(+), 6 deletions(-)

diff --git a/airflow/decorators/task_group.py b/airflow/decorators/task_group.py
index 2aa714be3b..1431a10452 100644
--- a/airflow/decorators/task_group.py
+++ b/airflow/decorators/task_group.py
@@ -182,6 +182,8 @@ def task_group(
     ui_color: str = "CornflowerBlue",
     ui_fgcolor: str = "#000",
     add_suffix_on_collision: bool = False,
+    setup: bool = False,
+    teardown: bool = False,
 ) -> Callable[[Callable[FParams, FReturn]], _TaskGroupFactory[FParams, 
FReturn]]:
     ...
 
diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index 8883fa659e..85249d3a98 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -92,11 +92,18 @@ class TaskGroup(DAGNode):
         ui_color: str = "CornflowerBlue",
         ui_fgcolor: str = "#000",
         add_suffix_on_collision: bool = False,
+        setup: bool = False,
+        teardown: bool = False,
     ):
         from airflow.models.dag import DagContext
 
+        if setup and teardown:
+            raise AirflowException("Cannot set both setup and teardown to 
True")
+
         self.prefix_group_id = prefix_group_id
         self.default_args = copy.deepcopy(default_args or {})
+        self.setup = setup
+        self.teardown = teardown
 
         dag = dag or DagContext.get_current_dag()
 
@@ -231,15 +238,37 @@ class TaskGroup(DAGNode):
             if task.children:
                 raise AirflowException("Cannot add a non-empty TaskGroup")
 
-        if SetupTeardownContext.is_setup:
+        is_setup, is_teardown = self._check_is_setup_teardown(task)
+
+        if SetupTeardownContext.is_setup or is_setup:
             if isinstance(task, AbstractOperator):
                 setattr(task, "_is_setup", True)
-        elif SetupTeardownContext.is_teardown:
+        elif SetupTeardownContext.is_teardown or is_teardown:
             if isinstance(task, AbstractOperator):
                 setattr(task, "_is_teardown", True)
 
         self.children[key] = task
 
+    def _check_is_setup_teardown(self, task_):
+        """Check if setup or teardown is set for the task"""
+        from airflow.models.abstractoperator import AbstractOperator
+
+        def _find_setup_teardown(tg):
+            setup, teardown = tg.setup, tg.teardown
+            while tg and tg.parent_group:
+                if setup or teardown:
+                    break
+                tg = tg.parent_group
+                setup, teardown = tg.setup, tg.teardown
+            return setup, teardown
+
+        if isinstance(task_, TaskGroup):
+            return _find_setup_teardown(task_)
+        if isinstance(task_, AbstractOperator):
+            tg = task_.task_group
+            return _find_setup_teardown(tg)
+        return False, False
+
     def _remove(self, task: DAGNode) -> None:
         key = task.node_id
 
diff --git a/tests/decorators/test_setup_teardown.py 
b/tests/decorators/test_setup_teardown.py
index 4dedd2421d..ec2d1773e4 100644
--- a/tests/decorators/test_setup_teardown.py
+++ b/tests/decorators/test_setup_teardown.py
@@ -17,7 +17,12 @@
 # under the License.
 from __future__ import annotations
 
+import pytest
+
+from airflow import AirflowException
 from airflow.decorators import setup, task, task_group, teardown
+from airflow.operators.bash import BashOperator
+from airflow.utils.task_group import TaskGroup
 
 
 class TestSetupTearDownTask:
@@ -59,8 +64,6 @@ class TestSetupTearDownTask:
         assert setup_task._is_setup
 
     def test_marking_operator_as_setup_task(self, dag_maker):
-        from airflow.operators.bash import BashOperator
-
         with dag_maker() as dag:
             BashOperator.as_setup(task_id="mytask", bash_command='echo "I am a 
setup task"')
 
@@ -91,7 +94,7 @@ class TestSetupTearDownTask:
         teardown_task = dag.task_group.children["mytask"]
         assert teardown_task._is_teardown
 
-    def test_setup_taskgroup(self, dag_maker):
+    def test_setup_taskgroup_decorator(self, dag_maker):
         @setup
         @task_group
         def mygroup():
@@ -110,7 +113,7 @@ class TestSetupTearDownTask:
         setup_task = setup_task_group.children["mygroup.mytask"]
         assert setup_task._is_setup
 
-    def test_teardown_taskgroup(self, dag_maker):
+    def test_teardown_taskgroup_decorator(self, dag_maker):
         @teardown
         @task_group
         def mygroup():
@@ -128,3 +131,306 @@ class TestSetupTearDownTask:
         assert len(teardown_task_group.children) == 1
         teardown_task = teardown_task_group.children["mygroup.mytask"]
         assert teardown_task._is_teardown
+
+    def test_setup_taskgroup_classic(self, dag_maker):
+        with dag_maker() as dag:
+            with TaskGroup("mygroup", setup=True):
+                BashOperator(task_id="mytask", bash_command='echo "I am a 
setup task"')
+
+        assert len(dag.task_group.children) == 1
+        setup_task_group = dag.task_group.children["mygroup"]
+        assert len(setup_task_group.children) == 1
+        setup_task = setup_task_group.children["mygroup.mytask"]
+        assert setup_task._is_setup
+
+    def test_teardown_taskgroup_classic(self, dag_maker):
+        with dag_maker() as dag:
+            with TaskGroup("mygroup", teardown=True):
+                BashOperator(task_id="mytask", bash_command='echo "I am a 
setup task"')
+
+        assert len(dag.task_group.children) == 1
+        teardown_task_group = dag.task_group.children["mygroup"]
+        assert len(teardown_task_group.children) == 1
+        teardown_task = teardown_task_group.children["mygroup.mytask"]
+        assert teardown_task._is_teardown
+
+    def test_setup_taskgroup_decorator_with_subgroup(self, dag_maker):
+        @setup
+        @task_group
+        def mygroup():
+            @task
+            def mytask():
+                print("I am a setup task")
+
+            @task_group
+            def subgroup():
+                @task
+                def mytask2():
+                    print("I am a task")
+
+                mytask2()
+
+            mytask()
+            subgroup()
+
+        with dag_maker() as dag:
+            mygroup()
+
+        assert len(dag.task_group.children) == 1
+        setup_task_group = dag.task_group.children["mygroup"]
+        assert len(setup_task_group.children) == 2
+        setup_task = setup_task_group.children["mygroup.mytask"]
+        assert setup_task._is_setup
+        subgroup_task_group = setup_task_group.children["mygroup.subgroup"]
+        assert len(subgroup_task_group.children) == 1
+        subgroup_task = 
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_setup
+
+    def test_teardown_taskgroup_decorator_with_subgroup(self, dag_maker):
+        @teardown
+        @task_group
+        def mygroup():
+            @task
+            def mytask():
+                print("I am a teardown task")
+
+            @task_group
+            def subgroup():
+                @task
+                def mytask2():
+                    print("I am a task")
+
+                mytask2()
+
+            mytask()
+            subgroup()
+
+        with dag_maker() as dag:
+            mygroup()
+
+        assert len(dag.task_group.children) == 1
+        teardown_task_group = dag.task_group.children["mygroup"]
+        assert len(teardown_task_group.children) == 2
+        teardown_task = teardown_task_group.children["mygroup.mytask"]
+        assert teardown_task._is_teardown
+        subgroup_task_group = teardown_task_group.children["mygroup.subgroup"]
+        assert len(subgroup_task_group.children) == 1
+        subgroup_task = 
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_teardown
+
+    def test_teardown_taskgroup_with_subgroup_classic(self, dag_maker):
+        with dag_maker() as dag:
+            with TaskGroup("mygroup", teardown=True):
+
+                @task
+                def mytask():
+                    print("I am a teardown task")
+
+                with TaskGroup("subgroup"):
+
+                    @task
+                    def mytask2():
+                        print("I am a teardown task")
+
+                    mytask2()
+
+                mytask()
+
+        assert len(dag.task_group.children) == 1
+        teardown_task_group = dag.task_group.children["mygroup"]
+        assert len(teardown_task_group.children) == 2
+        teardown_task = teardown_task_group.children["mygroup.mytask"]
+        assert teardown_task._is_teardown
+        subgroup_task_group = teardown_task_group.children["mygroup.subgroup"]
+        assert len(subgroup_task_group.children) == 1
+        subgroup_task = 
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_teardown
+
+    def test_setup_taskgroup_with_subgroup_classic(self, dag_maker):
+        with dag_maker() as dag:
+            with TaskGroup("mygroup", setup=True):
+
+                @task
+                def mytask():
+                    print("I am a setup task")
+
+                with TaskGroup("subgroup"):
+
+                    @task
+                    def mytask2():
+                        print("I am a setup task")
+
+                    mytask2()
+
+                mytask()
+
+        assert len(dag.task_group.children) == 1
+        setup_task_group = dag.task_group.children["mygroup"]
+        assert len(setup_task_group.children) == 2
+        setup_task = setup_task_group.children["mygroup.mytask"]
+        assert setup_task._is_setup
+        subgroup_task_group = setup_task_group.children["mygroup.subgroup"]
+        assert len(subgroup_task_group.children) == 1
+        subgroup_task = 
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_setup
+
+    def test_setup_taskgroup_with_subgroup_being_the_setup_decorated(self, 
dag_maker):
+        """Here, the subgroup is the setup taskgroup while the parent is not"""
+
+        with dag_maker() as dag:
+
+            @task_group
+            def mygroup():
+                @task
+                def mytask():
+                    print("I am a task")
+
+                @setup
+                @task_group
+                def subgroup():
+                    @task
+                    def mytask2():
+                        print("I am a setup task")
+
+                    mytask2()
+
+                subgroup()
+                mytask()
+
+            mygroup()
+        assert len(dag.task_group.children) == 1
+        normal_task_group = dag.task_group.children["mygroup"]
+        assert len(normal_task_group.children) == 2
+        normal_task = normal_task_group.children["mygroup.mytask"]
+        assert not hasattr(normal_task, "_is_setup")
+        setup_task_group = normal_task_group.children["mygroup.subgroup"]
+        assert len(setup_task_group.children) == 1
+        subgroup_task = setup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_setup
+
+    def 
test_teardown_taskgroup_with_subgroup_being_the_teardown_decorated(self, 
dag_maker):
+        """Here, the subgroup is the teardown taskgroup while the parent is 
not"""
+
+        @task_group
+        def mygroup():
+            @task
+            def mytask():
+                print("I am not a teardown task")
+
+            @teardown
+            @task_group
+            def subgroup():
+                @task
+                def mytask2():
+                    print("I am a teardown task")
+
+                mytask2()
+
+            mytask()
+            subgroup()
+
+        with dag_maker() as dag:
+            mygroup()
+
+        assert len(dag.task_group.children) == 1
+        normal_task_group = dag.task_group.children["mygroup"]
+        assert len(normal_task_group.children) == 2
+        normal_task = normal_task_group.children["mygroup.mytask"]
+        assert not hasattr(normal_task, "_is_teardown")
+        teardown_task_group = normal_task_group.children["mygroup.subgroup"]
+        assert len(teardown_task_group.children) == 1
+        subgroup_task = 
teardown_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_teardown
+
+    def test_setup_taskgroup_with_subgroup_being_the_setup_classic(self, 
dag_maker):
+        """Here, the subgroup is the setup taskgroup while the parent is not"""
+        with dag_maker() as dag:
+            with TaskGroup("mygroup"):
+
+                @task
+                def mytask():
+                    print("I am not a setup task")
+
+                with TaskGroup("subgroup", setup=True):
+
+                    @task
+                    def mytask2():
+                        print("I am a setup task")
+
+                    mytask2()
+
+                mytask()
+
+        assert len(dag.task_group.children) == 1
+        normal_task_group = dag.task_group.children["mygroup"]
+        assert len(normal_task_group.children) == 2
+        normal_task = normal_task_group.children["mygroup.mytask"]
+        assert not hasattr(normal_task, "_is_setup")
+        setup_task_group = normal_task_group.children["mygroup.subgroup"]
+        assert len(setup_task_group.children) == 1
+        subgroup_task = setup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_setup
+
+    def test_teardown_taskgroup_with_subgroup_being_the_teardown_classic(self, 
dag_maker):
+        """Here, the subgroup is the teardown taskgroup while the parent is 
not"""
+        with dag_maker() as dag:
+            with TaskGroup("mygroup"):
+
+                @task
+                def mytask():
+                    print("I am not a teardown task")
+
+                with TaskGroup("subgroup", teardown=True):
+
+                    @task
+                    def mytask2():
+                        print("I am a teardown task")
+
+                    mytask2()
+
+                mytask()
+
+        assert len(dag.task_group.children) == 1
+        normal_task_group = dag.task_group.children["mygroup"]
+        assert len(normal_task_group.children) == 2
+        normal_task = normal_task_group.children["mygroup.mytask"]
+        assert not hasattr(normal_task, "_is_teardown")
+        teardown_task_group = normal_task_group.children["mygroup.subgroup"]
+        assert len(teardown_task_group.children) == 1
+        subgroup_task = 
teardown_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_teardown
+
+    def test_setup_taskgroup_using_alternative_syntax(self, dag_maker):
+        """Here, the subgroup is the setup taskgroup while the parent is not"""
+        with dag_maker() as dag:
+            tg1 = TaskGroup("mygroup")
+
+            @task(task_group=tg1)
+            def mytask():
+                print("I am not a setup task")
+
+            tg2 = TaskGroup("subgroup", setup=True, parent_group=tg1)
+
+            @task(task_group=tg2)
+            def mytask2():
+                print("I am a setup task")
+
+            mytask2()
+
+            mytask()
+
+        assert len(dag.task_group.children) == 1
+        normal_task_group = dag.task_group.children["mygroup"]
+        assert len(normal_task_group.children) == 2
+        normal_task = normal_task_group.children["mygroup.mytask"]
+        assert not hasattr(normal_task, "_is_setup")
+        setup_task_group = normal_task_group.children["mygroup.subgroup"]
+        assert len(setup_task_group.children) == 1
+        subgroup_task = setup_task_group.children["mygroup.subgroup.mytask2"]
+        assert subgroup_task._is_setup
+
+    def test_setup_teardown_are_mutually_exclusive_on_taskgroup(self, 
dag_maker):
+        """Test that setup and teardown are mutually exclusive on TaskGroup"""
+        with dag_maker():
+            with pytest.raises(AirflowException, match="Cannot set both setup 
and teardown to True"):
+                TaskGroup("mygroup", setup=True, teardown=True)

Reply via email to