This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 41fb9b4c49 Deprecate `wait_for_completion` from
`EcsRegisterTaskDefinitionOperator` and `EcsDeregisterTaskDefinitionOperator`
(#31884)
41fb9b4c49 is described below
commit 41fb9b4c493dc74be7c21f68a2877cb2de97d983
Author: Raphaël Vandon <[email protected]>
AuthorDate: Fri Jun 16 15:39:30 2023 -0700
Deprecate `wait_for_completion` from `EcsRegisterTaskDefinitionOperator`
and `EcsDeregisterTaskDefinitionOperator` (#31884)
* Deprecate `wait_for_completion` from `EcsRegisterTaskDefinitionOperator`
and `EcsDeregisterTaskDefinitionOperator`
those operations are actually completed instantly.
the doc to deregister says it explicitly:
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/deregister-task-definition.html
> When you deregister a task definition revision, it’s immediately marked
as INACTIVE.
to register a task, it's less clear, but task definition states show no
transition states
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-definition-state.html
and experiments show that the task is always returned as active right after
having been created.
---
airflow/providers/amazon/aws/operators/ecs.py | 94 ++++++---------
airflow/providers/amazon/aws/waiters/ecs.json | 32 ------
tests/providers/amazon/aws/operators/test_ecs.py | 126 +++------------------
.../amazon/aws/waiters/test_custom_waiters.py | 31 -----
4 files changed, 46 insertions(+), 237 deletions(-)
diff --git a/airflow/providers/amazon/aws/operators/ecs.py
b/airflow/providers/amazon/aws/operators/ecs.py
index 3a62f389da..a72b15c18a 100644
--- a/airflow/providers/amazon/aws/operators/ecs.py
+++ b/airflow/providers/amazon/aws/operators/ecs.py
@@ -19,20 +19,20 @@ from __future__ import annotations
import re
import sys
+import warnings
from datetime import timedelta
from functools import cached_property
from typing import TYPE_CHECKING, Sequence
import boto3
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException,
AirflowProviderDeprecationWarning
from airflow.models import BaseOperator, XCom
from airflow.providers.amazon.aws.exceptions import EcsOperatorError,
EcsTaskFailToStart
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
from airflow.providers.amazon.aws.hooks.ecs import (
EcsClusterStates,
EcsHook,
- EcsTaskDefinitionStates,
EcsTaskLogFetcher,
should_retry_eni,
)
@@ -202,53 +202,39 @@ class
EcsDeregisterTaskDefinitionOperator(EcsBaseOperator):
:param task_definition: The family and revision (family:revision) or full
Amazon Resource Name (ARN)
of the task definition to deregister. If you use a family name, you
must specify a revision.
- :param wait_for_completion: If True, waits for creation of the cluster to
complete. (default: True)
- :param waiter_delay: The amount of time in seconds to wait between
attempts,
- if not set then the default waiter value will be used.
- :param waiter_max_attempts: The maximum number of attempts to be made,
- if not set then the default waiter value will be used.
"""
- template_fields: Sequence[str] = ("task_definition", "wait_for_completion")
+ template_fields: Sequence[str] = "task_definition"
def __init__(
self,
*,
task_definition: str,
- wait_for_completion: bool = True,
- waiter_delay: int | None = None,
- waiter_max_attempts: int | None = None,
**kwargs,
):
+ if "wait_for_completion" in kwargs or "waiter_delay" in kwargs or
"waiter_max_attempts" in kwargs:
+ warnings.warn(
+ "'wait_for_completion' and waiter related params have no
effect and are deprecated, "
+ "please remove them.",
+ AirflowProviderDeprecationWarning,
+ stacklevel=2,
+ )
+ # remove args to not trigger Invalid arguments exception
+ kwargs.pop("wait_for_completion", None)
+ kwargs.pop("waiter_delay", None)
+ kwargs.pop("waiter_max_attempts", None)
+
super().__init__(**kwargs)
self.task_definition = task_definition
- self.wait_for_completion = wait_for_completion
- self.waiter_delay = waiter_delay
- self.waiter_max_attempts = waiter_max_attempts
def execute(self, context: Context):
self.log.info("Deregistering task definition %s.",
self.task_definition)
result =
self.client.deregister_task_definition(taskDefinition=self.task_definition)
task_definition_details = result["taskDefinition"]
task_definition_arn = task_definition_details["taskDefinitionArn"]
- task_definition_state = task_definition_details.get("status")
-
- if task_definition_state == EcsTaskDefinitionStates.INACTIVE:
- # In some circumstances the ECS Task Definition is deleted
immediately,
- # so there is no reason to wait for completion.
- self.log.info("Task Definition %r in state: %r.",
task_definition_arn, task_definition_state)
- elif self.wait_for_completion:
- waiter = self.hook.get_waiter("task_definition_inactive")
- waiter.wait(
- taskDefinition=task_definition_arn,
- WaiterConfig=prune_dict(
- {
- "Delay": self.waiter_delay,
- "MaxAttempts": self.waiter_max_attempts,
- }
- ),
- )
-
+ self.log.info(
+ "Task Definition %r in state: %r.", task_definition_arn,
task_definition_details.get("status")
+ )
return task_definition_arn
@@ -264,18 +250,12 @@ class EcsRegisterTaskDefinitionOperator(EcsBaseOperator):
:param container_definitions: A list of container definitions in JSON
format that describe
the different containers that make up your task.
:param register_task_kwargs: Extra arguments for Register Task Definition.
- :param wait_for_completion: If True, waits for creation of the cluster to
complete. (default: True)
- :param waiter_delay: The amount of time in seconds to wait between
attempts,
- if not set then the default waiter value will be used.
- :param waiter_max_attempts: The maximum number of attempts to be made,
- if not set then the default waiter value will be used.
"""
template_fields: Sequence[str] = (
"family",
"container_definitions",
"register_task_kwargs",
- "wait_for_completion",
)
def __init__(
@@ -284,18 +264,24 @@ class EcsRegisterTaskDefinitionOperator(EcsBaseOperator):
family: str,
container_definitions: list[dict],
register_task_kwargs: dict | None = None,
- wait_for_completion: bool = True,
- waiter_delay: int | None = None,
- waiter_max_attempts: int | None = None,
**kwargs,
):
+ if "wait_for_completion" in kwargs or "waiter_delay" in kwargs or
"waiter_max_attempts" in kwargs:
+ warnings.warn(
+ "'wait_for_completion' and waiter related params have no
effect and are deprecated, "
+ "please remove them.",
+ AirflowProviderDeprecationWarning,
+ stacklevel=2,
+ )
+ # remove args to not trigger Invalid arguments exception
+ kwargs.pop("wait_for_completion", None)
+ kwargs.pop("waiter_delay", None)
+ kwargs.pop("waiter_max_attempts", None)
+
super().__init__(**kwargs)
self.family = family
self.container_definitions = container_definitions
self.register_task_kwargs = register_task_kwargs or {}
- self.wait_for_completion = wait_for_completion
- self.waiter_delay = waiter_delay
- self.waiter_max_attempts = waiter_max_attempts
def execute(self, context: Context):
self.log.info(
@@ -311,24 +297,10 @@ class EcsRegisterTaskDefinitionOperator(EcsBaseOperator):
)
task_definition_details = response["taskDefinition"]
task_definition_arn = task_definition_details["taskDefinitionArn"]
- task_definition_state = task_definition_details.get("status")
-
- if task_definition_state == EcsTaskDefinitionStates.ACTIVE:
- # In some circumstances the ECS Task Definition is created
immediately,
- # so there is no reason to wait for completion.
- self.log.info("Task Definition %r in state: %r.",
task_definition_arn, task_definition_state)
- elif self.wait_for_completion:
- waiter = self.hook.get_waiter("task_definition_active")
- waiter.wait(
- taskDefinition=task_definition_arn,
- WaiterConfig=prune_dict(
- {
- "Delay": self.waiter_delay,
- "MaxAttempts": self.waiter_max_attempts,
- }
- ),
- )
+ self.log.info(
+ "Task Definition %r in state: %r.", task_definition_arn,
task_definition_details.get("status")
+ )
context["ti"].xcom_push(key="task_definition_arn",
value=task_definition_arn)
return task_definition_arn
diff --git a/airflow/providers/amazon/aws/waiters/ecs.json
b/airflow/providers/amazon/aws/waiters/ecs.json
index c04679a34a..2d61050d5b 100644
--- a/airflow/providers/amazon/aws/waiters/ecs.json
+++ b/airflow/providers/amazon/aws/waiters/ecs.json
@@ -50,38 +50,6 @@
"argument": "failures[].reason"
}
]
- },
- "task_definition_active": {
- "operation": "DescribeTaskDefinition",
- "delay": 15,
- "maxAttempts": 60,
- "acceptors": [
- {
- "expected": "ACTIVE",
- "matcher": "path",
- "state": "success",
- "argument": "taskDefinition.status"
- },
- {
- "expected": "DELETE_IN_PROGRESS",
- "matcher": "path",
- "state": "failure",
- "argument": "taskDefinition.status"
- }
- ]
- },
- "task_definition_inactive": {
- "operation": "DescribeTaskDefinition",
- "delay": 15,
- "maxAttempts": 60,
- "acceptors": [
- {
- "expected": "INACTIVE",
- "matcher": "path",
- "state": "success",
- "argument": "taskDefinition.status"
- }
- ]
}
}
}
diff --git a/tests/providers/amazon/aws/operators/test_ecs.py
b/tests/providers/amazon/aws/operators/test_ecs.py
index ca23931e90..6366a879c2 100644
--- a/tests/providers/amazon/aws/operators/test_ecs.py
+++ b/tests/providers/amazon/aws/operators/test_ecs.py
@@ -24,7 +24,7 @@ from unittest import mock
import boto3
import pytest
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException,
AirflowProviderDeprecationWarning
from airflow.providers.amazon.aws.exceptions import EcsOperatorError,
EcsTaskFailToStart
from airflow.providers.amazon.aws.hooks.ecs import EcsHook
from airflow.providers.amazon.aws.operators.ecs import (
@@ -745,33 +745,7 @@ class TestEcsDeleteClusterOperator(EcsBaseTestCase):
class TestEcsDeregisterTaskDefinitionOperator(EcsBaseTestCase):
- @pytest.mark.parametrize("waiter_delay, waiter_max_attempts",
WAITERS_TEST_CASES)
- def test_execute_with_waiter(self, patch_hook_waiters, waiter_delay,
waiter_max_attempts):
- mocked_waiters = mock.MagicMock(name="MockedHookWaitersMethod")
- patch_hook_waiters.return_value = mocked_waiters
- op = EcsDeregisterTaskDefinitionOperator(
- task_id="task",
- task_definition=TASK_DEFINITION_NAME,
- wait_for_completion=True,
- waiter_delay=waiter_delay,
- waiter_max_attempts=waiter_max_attempts,
- )
- with mock.patch.object(self.client, "deregister_task_definition") as
mock_client_method:
- result = op.execute({})
-
mock_client_method.assert_called_once_with(taskDefinition=TASK_DEFINITION_NAME)
- patch_hook_waiters.assert_called_once_with("task_definition_inactive")
-
- expected_waiter_config = {}
- if waiter_delay:
- expected_waiter_config["Delay"] = waiter_delay
- if waiter_max_attempts:
- expected_waiter_config["MaxAttempts"] = waiter_max_attempts
- mocked_waiters.wait.assert_called_once_with(
- taskDefinition=mock.ANY, WaiterConfig=expected_waiter_config
- )
- assert result is not None
-
- def test_execute_immediate_delete(self, patch_hook_waiters):
+ def test_execute_immediate_delete(self):
"""Test if task definition deleted during initial request."""
op = EcsDeregisterTaskDefinitionOperator(
task_id="task", task_definition=TASK_DEFINITION_NAME,
wait_for_completion=True
@@ -782,69 +756,15 @@ class
TestEcsDeregisterTaskDefinitionOperator(EcsBaseTestCase):
}
result = op.execute({})
mock_client_method.assert_called_once_with(taskDefinition=TASK_DEFINITION_NAME)
- patch_hook_waiters.assert_not_called()
assert result == "foo-bar"
- def test_execute_without_waiter(self, patch_hook_waiters):
- op = EcsDeregisterTaskDefinitionOperator(
- task_id="task", task_definition=TASK_DEFINITION_NAME,
wait_for_completion=False
- )
- with mock.patch.object(self.client, "deregister_task_definition") as
mock_client_method:
- result = op.execute({})
-
mock_client_method.assert_called_once_with(taskDefinition=TASK_DEFINITION_NAME)
- patch_hook_waiters.assert_not_called()
- assert result is not None
+ def test_deprecation(self):
+ with pytest.warns(AirflowProviderDeprecationWarning):
+ EcsDeregisterTaskDefinitionOperator(task_id="id",
task_definition="def", wait_for_completion=True)
class TestEcsRegisterTaskDefinitionOperator(EcsBaseTestCase):
- @pytest.mark.parametrize("waiter_delay, waiter_max_attempts",
WAITERS_TEST_CASES)
- def test_execute_with_waiter(self, patch_hook_waiters, waiter_delay,
waiter_max_attempts):
- mock_ti = mock.MagicMock(name="MockedTaskInstance")
- mocked_waiters = mock.MagicMock(name="MockedHookWaitersMethod")
- patch_hook_waiters.return_value = mocked_waiters
- expected_task_definition_config = {
- "family": "family_name",
- "containerDefinitions": [
- {
- "name": CONTAINER_NAME,
- "image": "ubuntu",
- "workingDirectory": "/usr/bin",
- "entryPoint": ["sh", "-c"],
- "command": ["ls"],
- }
- ],
- "cpu": "256",
- "memory": "512",
- "networkMode": "awsvpc",
- }
- op = EcsRegisterTaskDefinitionOperator(
- task_id="task",
- **TASK_DEFINITION_CONFIG,
- wait_for_completion=True,
- waiter_delay=waiter_delay,
- waiter_max_attempts=waiter_max_attempts,
- )
- with mock.patch.object(self.client, "register_task_definition") as
mock_client_method:
- result = op.execute({"ti": mock_ti})
-
mock_client_method.assert_called_once_with(**expected_task_definition_config)
- patch_hook_waiters.assert_called_once_with("task_definition_active")
- mock_ti.xcom_push.assert_called_once_with(key="task_definition_arn",
value=mock.ANY)
-
- expected_waiter_config = {}
- if waiter_delay:
- expected_waiter_config["Delay"] = waiter_delay
- if waiter_max_attempts:
- expected_waiter_config["MaxAttempts"] = waiter_max_attempts
- mocked_waiters.wait.assert_called_once_with(
- taskDefinition=mock.ANY, WaiterConfig=expected_waiter_config
- )
- mocked_waiters.wait.assert_called_once_with(
- taskDefinition=mock.ANY, WaiterConfig=expected_waiter_config
- )
-
- assert result is not None
-
- def test_execute_immediate_create(self, patch_hook_waiters):
+ def test_execute_immediate_create(self):
"""Test if task definition created during initial request."""
mock_ti = mock.MagicMock(name="MockedTaskInstance")
expected_task_definition_config = {
@@ -863,39 +783,19 @@ class
TestEcsRegisterTaskDefinitionOperator(EcsBaseTestCase):
"networkMode": "awsvpc",
}
op = EcsRegisterTaskDefinitionOperator(task_id="task",
**TASK_DEFINITION_CONFIG)
+
with mock.patch.object(self.client, "register_task_definition") as
mock_client_method:
mock_client_method.return_value = {
"taskDefinition": {"status": "ACTIVE", "taskDefinitionArn":
"foo-bar"}
}
result = op.execute({"ti": mock_ti})
mock_client_method.assert_called_once_with(**expected_task_definition_config)
- patch_hook_waiters.assert_not_called()
+
mock_ti.xcom_push.assert_called_once_with(key="task_definition_arn",
value="foo-bar")
assert result == "foo-bar"
- def test_execute_without_waiter(self, patch_hook_waiters):
- mock_context = mock.MagicMock()
- expected_task_definition_config = {
- "family": "family_name",
- "containerDefinitions": [
- {
- "name": CONTAINER_NAME,
- "image": "ubuntu",
- "workingDirectory": "/usr/bin",
- "entryPoint": ["sh", "-c"],
- "command": ["ls"],
- }
- ],
- "cpu": "256",
- "memory": "512",
- "networkMode": "awsvpc",
- }
-
- op = EcsRegisterTaskDefinitionOperator(
- task_id="task", **TASK_DEFINITION_CONFIG, wait_for_completion=False
- )
- with mock.patch.object(self.client, "register_task_definition") as
mock_client_method:
- result = op.execute(mock_context)
-
mock_client_method.assert_called_once_with(**expected_task_definition_config)
- patch_hook_waiters.assert_not_called()
- assert result is not None
+ def test_deprecation(self):
+ with pytest.warns(AirflowProviderDeprecationWarning):
+ EcsRegisterTaskDefinitionOperator(
+ task_id="id", wait_for_completion=True,
**TASK_DEFINITION_CONFIG
+ )
diff --git a/tests/providers/amazon/aws/waiters/test_custom_waiters.py
b/tests/providers/amazon/aws/waiters/test_custom_waiters.py
index 227ca6be42..d02c9c49e8 100644
--- a/tests/providers/amazon/aws/waiters/test_custom_waiters.py
+++ b/tests/providers/amazon/aws/waiters/test_custom_waiters.py
@@ -129,8 +129,6 @@ class TestCustomECSServiceWaiters:
hook_waiters = EcsHook(aws_conn_id=None).list_waiters()
assert "cluster_active" in hook_waiters
assert "cluster_inactive" in hook_waiters
- assert "task_definition_active" in hook_waiters
- assert "task_definition_inactive" in hook_waiters
@staticmethod
def describe_clusters(
@@ -222,35 +220,6 @@ class TestCustomECSServiceWaiters:
}
}
- def test_task_definition_active(self, mock_describe_task_definition):
- """Test task definition reach active state during creation."""
- mock_describe_task_definition.side_effect = [
- self.describe_task_definition(EcsTaskDefinitionStates.INACTIVE),
- self.describe_task_definition(EcsTaskDefinitionStates.INACTIVE),
- self.describe_task_definition(EcsTaskDefinitionStates.ACTIVE),
- ]
- waiter = EcsHook(aws_conn_id=None).get_waiter("task_definition_active")
- waiter.wait(taskDefinition="spam-egg", WaiterConfig={"Delay": 0.01,
"MaxAttempts": 3})
-
- def test_task_definition_failure(self, mock_describe_task_definition):
- """Test task definition reach delete in progress state during
creation."""
- mock_describe_task_definition.side_effect = [
-
self.describe_task_definition(EcsTaskDefinitionStates.DELETE_IN_PROGRESS),
- ]
- waiter = EcsHook(aws_conn_id=None).get_waiter("task_definition_active")
- with pytest.raises(WaiterError, match='matched expected path:
"DELETE_IN_PROGRESS"'):
- waiter.wait(taskDefinition="spam-egg", WaiterConfig={"Delay":
0.01, "MaxAttempts": 1})
-
- def test_task_definition_inactive(self, mock_describe_task_definition):
- """Test task definition reach inactive state during deletion."""
- mock_describe_task_definition.side_effect = [
- self.describe_task_definition(EcsTaskDefinitionStates.ACTIVE),
- self.describe_task_definition(EcsTaskDefinitionStates.ACTIVE),
- self.describe_task_definition(EcsTaskDefinitionStates.INACTIVE),
- ]
- waiter =
EcsHook(aws_conn_id=None).get_waiter("task_definition_inactive")
- waiter.wait(taskDefinition="spam-egg", WaiterConfig={"Delay": 0.01,
"MaxAttempts": 3})
-
class TestCustomDynamoDBServiceWaiters:
"""Test waiters from ``amazon/aws/waiters/dynamodb.json``."""