This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 41fb9b4c49 Deprecate `wait_for_completion` from 
`EcsRegisterTaskDefinitionOperator` and `EcsDeregisterTaskDefinitionOperator` 
(#31884)
41fb9b4c49 is described below

commit 41fb9b4c493dc74be7c21f68a2877cb2de97d983
Author: Raphaël Vandon <[email protected]>
AuthorDate: Fri Jun 16 15:39:30 2023 -0700

    Deprecate `wait_for_completion` from `EcsRegisterTaskDefinitionOperator` 
and `EcsDeregisterTaskDefinitionOperator` (#31884)
    
    * Deprecate `wait_for_completion` from `EcsRegisterTaskDefinitionOperator` 
and `EcsDeregisterTaskDefinitionOperator`
    
    those operations are actually completed instantly.
    
    the doc to deregister says it explicitly:
    
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/deregister-task-definition.html
    > When you deregister a task definition revision, it’s immediately marked 
as INACTIVE.
    
    to register a task, it's less clear, but task definition states show no 
transition states
    
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-definition-state.html
    and experiments show that the task is always returned as active right after 
having been created.
---
 airflow/providers/amazon/aws/operators/ecs.py      |  94 ++++++---------
 airflow/providers/amazon/aws/waiters/ecs.json      |  32 ------
 tests/providers/amazon/aws/operators/test_ecs.py   | 126 +++------------------
 .../amazon/aws/waiters/test_custom_waiters.py      |  31 -----
 4 files changed, 46 insertions(+), 237 deletions(-)

diff --git a/airflow/providers/amazon/aws/operators/ecs.py 
b/airflow/providers/amazon/aws/operators/ecs.py
index 3a62f389da..a72b15c18a 100644
--- a/airflow/providers/amazon/aws/operators/ecs.py
+++ b/airflow/providers/amazon/aws/operators/ecs.py
@@ -19,20 +19,20 @@ from __future__ import annotations
 
 import re
 import sys
+import warnings
 from datetime import timedelta
 from functools import cached_property
 from typing import TYPE_CHECKING, Sequence
 
 import boto3
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, 
AirflowProviderDeprecationWarning
 from airflow.models import BaseOperator, XCom
 from airflow.providers.amazon.aws.exceptions import EcsOperatorError, 
EcsTaskFailToStart
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 from airflow.providers.amazon.aws.hooks.ecs import (
     EcsClusterStates,
     EcsHook,
-    EcsTaskDefinitionStates,
     EcsTaskLogFetcher,
     should_retry_eni,
 )
@@ -202,53 +202,39 @@ class 
EcsDeregisterTaskDefinitionOperator(EcsBaseOperator):
 
     :param task_definition: The family and revision (family:revision) or full 
Amazon Resource Name (ARN)
         of the task definition to deregister. If you use a family name, you 
must specify a revision.
-    :param wait_for_completion: If True, waits for creation of the cluster to 
complete. (default: True)
-    :param waiter_delay: The amount of time in seconds to wait between 
attempts,
-        if not set then the default waiter value will be used.
-    :param waiter_max_attempts: The maximum number of attempts to be made,
-        if not set then the default waiter value will be used.
     """
 
-    template_fields: Sequence[str] = ("task_definition", "wait_for_completion")
+    template_fields: Sequence[str] = "task_definition"
 
     def __init__(
         self,
         *,
         task_definition: str,
-        wait_for_completion: bool = True,
-        waiter_delay: int | None = None,
-        waiter_max_attempts: int | None = None,
         **kwargs,
     ):
+        if "wait_for_completion" in kwargs or "waiter_delay" in kwargs or 
"waiter_max_attempts" in kwargs:
+            warnings.warn(
+                "'wait_for_completion' and waiter related params have no 
effect and are deprecated, "
+                "please remove them.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
+            # remove args to not trigger Invalid arguments exception
+            kwargs.pop("wait_for_completion", None)
+            kwargs.pop("waiter_delay", None)
+            kwargs.pop("waiter_max_attempts", None)
+
         super().__init__(**kwargs)
         self.task_definition = task_definition
-        self.wait_for_completion = wait_for_completion
-        self.waiter_delay = waiter_delay
-        self.waiter_max_attempts = waiter_max_attempts
 
     def execute(self, context: Context):
         self.log.info("Deregistering task definition %s.", 
self.task_definition)
         result = 
self.client.deregister_task_definition(taskDefinition=self.task_definition)
         task_definition_details = result["taskDefinition"]
         task_definition_arn = task_definition_details["taskDefinitionArn"]
-        task_definition_state = task_definition_details.get("status")
-
-        if task_definition_state == EcsTaskDefinitionStates.INACTIVE:
-            # In some circumstances the ECS Task Definition is deleted 
immediately,
-            # so there is no reason to wait for completion.
-            self.log.info("Task Definition %r in state: %r.", 
task_definition_arn, task_definition_state)
-        elif self.wait_for_completion:
-            waiter = self.hook.get_waiter("task_definition_inactive")
-            waiter.wait(
-                taskDefinition=task_definition_arn,
-                WaiterConfig=prune_dict(
-                    {
-                        "Delay": self.waiter_delay,
-                        "MaxAttempts": self.waiter_max_attempts,
-                    }
-                ),
-            )
-
+        self.log.info(
+            "Task Definition %r in state: %r.", task_definition_arn, 
task_definition_details.get("status")
+        )
         return task_definition_arn
 
 
@@ -264,18 +250,12 @@ class EcsRegisterTaskDefinitionOperator(EcsBaseOperator):
     :param container_definitions: A list of container definitions in JSON 
format that describe
         the different containers that make up your task.
     :param register_task_kwargs: Extra arguments for Register Task Definition.
-    :param wait_for_completion: If True, waits for creation of the cluster to 
complete. (default: True)
-    :param waiter_delay: The amount of time in seconds to wait between 
attempts,
-        if not set then the default waiter value will be used.
-    :param waiter_max_attempts: The maximum number of attempts to be made,
-        if not set then the default waiter value will be used.
     """
 
     template_fields: Sequence[str] = (
         "family",
         "container_definitions",
         "register_task_kwargs",
-        "wait_for_completion",
     )
 
     def __init__(
@@ -284,18 +264,24 @@ class EcsRegisterTaskDefinitionOperator(EcsBaseOperator):
         family: str,
         container_definitions: list[dict],
         register_task_kwargs: dict | None = None,
-        wait_for_completion: bool = True,
-        waiter_delay: int | None = None,
-        waiter_max_attempts: int | None = None,
         **kwargs,
     ):
+        if "wait_for_completion" in kwargs or "waiter_delay" in kwargs or 
"waiter_max_attempts" in kwargs:
+            warnings.warn(
+                "'wait_for_completion' and waiter related params have no 
effect and are deprecated, "
+                "please remove them.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
+            # remove args to not trigger Invalid arguments exception
+            kwargs.pop("wait_for_completion", None)
+            kwargs.pop("waiter_delay", None)
+            kwargs.pop("waiter_max_attempts", None)
+
         super().__init__(**kwargs)
         self.family = family
         self.container_definitions = container_definitions
         self.register_task_kwargs = register_task_kwargs or {}
-        self.wait_for_completion = wait_for_completion
-        self.waiter_delay = waiter_delay
-        self.waiter_max_attempts = waiter_max_attempts
 
     def execute(self, context: Context):
         self.log.info(
@@ -311,24 +297,10 @@ class EcsRegisterTaskDefinitionOperator(EcsBaseOperator):
         )
         task_definition_details = response["taskDefinition"]
         task_definition_arn = task_definition_details["taskDefinitionArn"]
-        task_definition_state = task_definition_details.get("status")
-
-        if task_definition_state == EcsTaskDefinitionStates.ACTIVE:
-            # In some circumstances the ECS Task Definition is created 
immediately,
-            # so there is no reason to wait for completion.
-            self.log.info("Task Definition %r in state: %r.", 
task_definition_arn, task_definition_state)
-        elif self.wait_for_completion:
-            waiter = self.hook.get_waiter("task_definition_active")
-            waiter.wait(
-                taskDefinition=task_definition_arn,
-                WaiterConfig=prune_dict(
-                    {
-                        "Delay": self.waiter_delay,
-                        "MaxAttempts": self.waiter_max_attempts,
-                    }
-                ),
-            )
 
+        self.log.info(
+            "Task Definition %r in state: %r.", task_definition_arn, 
task_definition_details.get("status")
+        )
         context["ti"].xcom_push(key="task_definition_arn", 
value=task_definition_arn)
         return task_definition_arn
 
diff --git a/airflow/providers/amazon/aws/waiters/ecs.json 
b/airflow/providers/amazon/aws/waiters/ecs.json
index c04679a34a..2d61050d5b 100644
--- a/airflow/providers/amazon/aws/waiters/ecs.json
+++ b/airflow/providers/amazon/aws/waiters/ecs.json
@@ -50,38 +50,6 @@
                   "argument": "failures[].reason"
                 }
             ]
-        },
-        "task_definition_active": {
-            "operation": "DescribeTaskDefinition",
-            "delay": 15,
-            "maxAttempts": 60,
-            "acceptors": [
-                {
-                    "expected": "ACTIVE",
-                    "matcher": "path",
-                    "state": "success",
-                    "argument": "taskDefinition.status"
-                },
-                {
-                    "expected": "DELETE_IN_PROGRESS",
-                    "matcher": "path",
-                    "state": "failure",
-                    "argument": "taskDefinition.status"
-                }
-            ]
-        },
-        "task_definition_inactive": {
-            "operation": "DescribeTaskDefinition",
-            "delay": 15,
-            "maxAttempts": 60,
-            "acceptors": [
-                {
-                    "expected": "INACTIVE",
-                    "matcher": "path",
-                    "state": "success",
-                    "argument": "taskDefinition.status"
-                }
-            ]
         }
     }
 }
diff --git a/tests/providers/amazon/aws/operators/test_ecs.py 
b/tests/providers/amazon/aws/operators/test_ecs.py
index ca23931e90..6366a879c2 100644
--- a/tests/providers/amazon/aws/operators/test_ecs.py
+++ b/tests/providers/amazon/aws/operators/test_ecs.py
@@ -24,7 +24,7 @@ from unittest import mock
 import boto3
 import pytest
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, 
AirflowProviderDeprecationWarning
 from airflow.providers.amazon.aws.exceptions import EcsOperatorError, 
EcsTaskFailToStart
 from airflow.providers.amazon.aws.hooks.ecs import EcsHook
 from airflow.providers.amazon.aws.operators.ecs import (
@@ -745,33 +745,7 @@ class TestEcsDeleteClusterOperator(EcsBaseTestCase):
 
 
 class TestEcsDeregisterTaskDefinitionOperator(EcsBaseTestCase):
-    @pytest.mark.parametrize("waiter_delay, waiter_max_attempts", 
WAITERS_TEST_CASES)
-    def test_execute_with_waiter(self, patch_hook_waiters, waiter_delay, 
waiter_max_attempts):
-        mocked_waiters = mock.MagicMock(name="MockedHookWaitersMethod")
-        patch_hook_waiters.return_value = mocked_waiters
-        op = EcsDeregisterTaskDefinitionOperator(
-            task_id="task",
-            task_definition=TASK_DEFINITION_NAME,
-            wait_for_completion=True,
-            waiter_delay=waiter_delay,
-            waiter_max_attempts=waiter_max_attempts,
-        )
-        with mock.patch.object(self.client, "deregister_task_definition") as 
mock_client_method:
-            result = op.execute({})
-            
mock_client_method.assert_called_once_with(taskDefinition=TASK_DEFINITION_NAME)
-        patch_hook_waiters.assert_called_once_with("task_definition_inactive")
-
-        expected_waiter_config = {}
-        if waiter_delay:
-            expected_waiter_config["Delay"] = waiter_delay
-        if waiter_max_attempts:
-            expected_waiter_config["MaxAttempts"] = waiter_max_attempts
-        mocked_waiters.wait.assert_called_once_with(
-            taskDefinition=mock.ANY, WaiterConfig=expected_waiter_config
-        )
-        assert result is not None
-
-    def test_execute_immediate_delete(self, patch_hook_waiters):
+    def test_execute_immediate_delete(self):
         """Test if task definition deleted during initial request."""
         op = EcsDeregisterTaskDefinitionOperator(
             task_id="task", task_definition=TASK_DEFINITION_NAME, 
wait_for_completion=True
@@ -782,69 +756,15 @@ class 
TestEcsDeregisterTaskDefinitionOperator(EcsBaseTestCase):
             }
             result = op.execute({})
             
mock_client_method.assert_called_once_with(taskDefinition=TASK_DEFINITION_NAME)
-        patch_hook_waiters.assert_not_called()
         assert result == "foo-bar"
 
-    def test_execute_without_waiter(self, patch_hook_waiters):
-        op = EcsDeregisterTaskDefinitionOperator(
-            task_id="task", task_definition=TASK_DEFINITION_NAME, 
wait_for_completion=False
-        )
-        with mock.patch.object(self.client, "deregister_task_definition") as 
mock_client_method:
-            result = op.execute({})
-            
mock_client_method.assert_called_once_with(taskDefinition=TASK_DEFINITION_NAME)
-        patch_hook_waiters.assert_not_called()
-        assert result is not None
+    def test_deprecation(self):
+        with pytest.warns(AirflowProviderDeprecationWarning):
+            EcsDeregisterTaskDefinitionOperator(task_id="id", 
task_definition="def", wait_for_completion=True)
 
 
 class TestEcsRegisterTaskDefinitionOperator(EcsBaseTestCase):
-    @pytest.mark.parametrize("waiter_delay, waiter_max_attempts", 
WAITERS_TEST_CASES)
-    def test_execute_with_waiter(self, patch_hook_waiters, waiter_delay, 
waiter_max_attempts):
-        mock_ti = mock.MagicMock(name="MockedTaskInstance")
-        mocked_waiters = mock.MagicMock(name="MockedHookWaitersMethod")
-        patch_hook_waiters.return_value = mocked_waiters
-        expected_task_definition_config = {
-            "family": "family_name",
-            "containerDefinitions": [
-                {
-                    "name": CONTAINER_NAME,
-                    "image": "ubuntu",
-                    "workingDirectory": "/usr/bin",
-                    "entryPoint": ["sh", "-c"],
-                    "command": ["ls"],
-                }
-            ],
-            "cpu": "256",
-            "memory": "512",
-            "networkMode": "awsvpc",
-        }
-        op = EcsRegisterTaskDefinitionOperator(
-            task_id="task",
-            **TASK_DEFINITION_CONFIG,
-            wait_for_completion=True,
-            waiter_delay=waiter_delay,
-            waiter_max_attempts=waiter_max_attempts,
-        )
-        with mock.patch.object(self.client, "register_task_definition") as 
mock_client_method:
-            result = op.execute({"ti": mock_ti})
-            
mock_client_method.assert_called_once_with(**expected_task_definition_config)
-        patch_hook_waiters.assert_called_once_with("task_definition_active")
-        mock_ti.xcom_push.assert_called_once_with(key="task_definition_arn", 
value=mock.ANY)
-
-        expected_waiter_config = {}
-        if waiter_delay:
-            expected_waiter_config["Delay"] = waiter_delay
-        if waiter_max_attempts:
-            expected_waiter_config["MaxAttempts"] = waiter_max_attempts
-        mocked_waiters.wait.assert_called_once_with(
-            taskDefinition=mock.ANY, WaiterConfig=expected_waiter_config
-        )
-        mocked_waiters.wait.assert_called_once_with(
-            taskDefinition=mock.ANY, WaiterConfig=expected_waiter_config
-        )
-
-        assert result is not None
-
-    def test_execute_immediate_create(self, patch_hook_waiters):
+    def test_execute_immediate_create(self):
         """Test if task definition created during initial request."""
         mock_ti = mock.MagicMock(name="MockedTaskInstance")
         expected_task_definition_config = {
@@ -863,39 +783,19 @@ class 
TestEcsRegisterTaskDefinitionOperator(EcsBaseTestCase):
             "networkMode": "awsvpc",
         }
         op = EcsRegisterTaskDefinitionOperator(task_id="task", 
**TASK_DEFINITION_CONFIG)
+
         with mock.patch.object(self.client, "register_task_definition") as 
mock_client_method:
             mock_client_method.return_value = {
                 "taskDefinition": {"status": "ACTIVE", "taskDefinitionArn": 
"foo-bar"}
             }
             result = op.execute({"ti": mock_ti})
             
mock_client_method.assert_called_once_with(**expected_task_definition_config)
-        patch_hook_waiters.assert_not_called()
+
         mock_ti.xcom_push.assert_called_once_with(key="task_definition_arn", 
value="foo-bar")
         assert result == "foo-bar"
 
-    def test_execute_without_waiter(self, patch_hook_waiters):
-        mock_context = mock.MagicMock()
-        expected_task_definition_config = {
-            "family": "family_name",
-            "containerDefinitions": [
-                {
-                    "name": CONTAINER_NAME,
-                    "image": "ubuntu",
-                    "workingDirectory": "/usr/bin",
-                    "entryPoint": ["sh", "-c"],
-                    "command": ["ls"],
-                }
-            ],
-            "cpu": "256",
-            "memory": "512",
-            "networkMode": "awsvpc",
-        }
-
-        op = EcsRegisterTaskDefinitionOperator(
-            task_id="task", **TASK_DEFINITION_CONFIG, wait_for_completion=False
-        )
-        with mock.patch.object(self.client, "register_task_definition") as 
mock_client_method:
-            result = op.execute(mock_context)
-            
mock_client_method.assert_called_once_with(**expected_task_definition_config)
-        patch_hook_waiters.assert_not_called()
-        assert result is not None
+    def test_deprecation(self):
+        with pytest.warns(AirflowProviderDeprecationWarning):
+            EcsRegisterTaskDefinitionOperator(
+                task_id="id", wait_for_completion=True, 
**TASK_DEFINITION_CONFIG
+            )
diff --git a/tests/providers/amazon/aws/waiters/test_custom_waiters.py 
b/tests/providers/amazon/aws/waiters/test_custom_waiters.py
index 227ca6be42..d02c9c49e8 100644
--- a/tests/providers/amazon/aws/waiters/test_custom_waiters.py
+++ b/tests/providers/amazon/aws/waiters/test_custom_waiters.py
@@ -129,8 +129,6 @@ class TestCustomECSServiceWaiters:
         hook_waiters = EcsHook(aws_conn_id=None).list_waiters()
         assert "cluster_active" in hook_waiters
         assert "cluster_inactive" in hook_waiters
-        assert "task_definition_active" in hook_waiters
-        assert "task_definition_inactive" in hook_waiters
 
     @staticmethod
     def describe_clusters(
@@ -222,35 +220,6 @@ class TestCustomECSServiceWaiters:
             }
         }
 
-    def test_task_definition_active(self, mock_describe_task_definition):
-        """Test task definition reach active state during creation."""
-        mock_describe_task_definition.side_effect = [
-            self.describe_task_definition(EcsTaskDefinitionStates.INACTIVE),
-            self.describe_task_definition(EcsTaskDefinitionStates.INACTIVE),
-            self.describe_task_definition(EcsTaskDefinitionStates.ACTIVE),
-        ]
-        waiter = EcsHook(aws_conn_id=None).get_waiter("task_definition_active")
-        waiter.wait(taskDefinition="spam-egg", WaiterConfig={"Delay": 0.01, 
"MaxAttempts": 3})
-
-    def test_task_definition_failure(self, mock_describe_task_definition):
-        """Test task definition reach delete in progress state during 
creation."""
-        mock_describe_task_definition.side_effect = [
-            
self.describe_task_definition(EcsTaskDefinitionStates.DELETE_IN_PROGRESS),
-        ]
-        waiter = EcsHook(aws_conn_id=None).get_waiter("task_definition_active")
-        with pytest.raises(WaiterError, match='matched expected path: 
"DELETE_IN_PROGRESS"'):
-            waiter.wait(taskDefinition="spam-egg", WaiterConfig={"Delay": 
0.01, "MaxAttempts": 1})
-
-    def test_task_definition_inactive(self, mock_describe_task_definition):
-        """Test task definition reach inactive state during deletion."""
-        mock_describe_task_definition.side_effect = [
-            self.describe_task_definition(EcsTaskDefinitionStates.ACTIVE),
-            self.describe_task_definition(EcsTaskDefinitionStates.ACTIVE),
-            self.describe_task_definition(EcsTaskDefinitionStates.INACTIVE),
-        ]
-        waiter = 
EcsHook(aws_conn_id=None).get_waiter("task_definition_inactive")
-        waiter.wait(taskDefinition="spam-egg", WaiterConfig={"Delay": 0.01, 
"MaxAttempts": 3})
-
 
 class TestCustomDynamoDBServiceWaiters:
     """Test waiters from ``amazon/aws/waiters/dynamodb.json``."""

Reply via email to