This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ce4236f365c Add DMS Serverless Operators (#43988)
ce4236f365c is described below
commit ce4236f365c3e8e45262b04ef0ce596907043702
Author: ellisms <[email protected]>
AuthorDate: Tue Dec 17 10:04:49 2024 -0500
Add DMS Serverless Operators (#43988)
* Adding DMS serverless operators
---------
Co-authored-by: Niko Oliveira <[email protected]>
Co-authored-by: mse139 <[email protected]>
Co-authored-by: Elad Kalif <[email protected]>
---
.../operators/dms.rst | 86 +++
.../src/airflow/providers/amazon/aws/hooks/dms.py | 161 ++++++
.../airflow/providers/amazon/aws/operators/dms.py | 531 +++++++++++++++++-
.../airflow/providers/amazon/aws/triggers/dms.py | 221 ++++++++
.../airflow/providers/amazon/aws/waiters/dms.json | 88 +++
.../src/airflow/providers/amazon/provider.yaml | 4 +
providers/tests/amazon/aws/hooks/test_dms.py | 297 +++++++++++
providers/tests/amazon/aws/operators/test_dms.py | 592 ++++++++++++++++++++-
providers/tests/amazon/aws/triggers/test_dms.py | 188 +++++++
providers/tests/amazon/aws/waiters/test_dms.py | 139 +++++
.../system/amazon/aws/example_dms_serverless.py | 481 +++++++++++++++++
11 files changed, 2785 insertions(+), 3 deletions(-)
diff --git a/docs/apache-airflow-providers-amazon/operators/dms.rst
b/docs/apache-airflow-providers-amazon/operators/dms.rst
index b80fb9e2b3a..fc2a7da2ded 100644
--- a/docs/apache-airflow-providers-amazon/operators/dms.rst
+++ b/docs/apache-airflow-providers-amazon/operators/dms.rst
@@ -114,6 +114,92 @@ To delete a replication task you can use
:start-after: [START howto_operator_dms_delete_task]
:end-before: [END howto_operator_dms_delete_task]
+.. _howto/operator:DmsCreateReplicationConfigOperator:
+
+Create a serverless replication config
+======================================
+
+To create a serverless replication config use
+:class:`~airflow.providers.amazon.aws.operators.dms.DmsCreateReplicationConfigOperator`.
+
+.. exampleinclude::
/../../providers/tests/system/amazon/aws/example_dms_serverless.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_dms_create_replication_config]
+ :end-before: [END howto_operator_dms_create_replication_config]
+
+.. _howto/operator:DmsDescribeReplicationConfigsOperator:
+
+Describe a serverless replication config
+========================================
+
+To describe a serverless replication config use
+:class:`~airflow.providers.amazon.aws.operators.dms.DmsDescribeReplicationConfigsOperator`.
+
+.. exampleinclude::
/../../providers/tests/system/amazon/aws/example_dms_serverless.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_dms_describe_replication_config]
+ :end-before: [END howto_operator_dms_describe_replication_config]
+
+.. _howto/operator:DmsStartReplicationOperator:
+
+Start a serverless replication
+==============================
+
+To start a serverless replication use
+:class:`~airflow.providers.amazon.aws.operators.dms.DmsStartReplicationOperator`.
+
+.. exampleinclude::
/../../providers/tests/system/amazon/aws/example_dms_serverless.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_dms_serverless_start_replication]
+ :end-before: [END howto_operator_dms_serverless_start_replication]
+
+.. _howto/operator:DmsStopReplicationOperator:
+
+Stop a serverless replication
+==============================
+
+To stop a serverless replication use
+:class:`~airflow.providers.amazon.aws.operators.dms.DmsStopReplicationOperator`.
+
+.. exampleinclude::
/../../providers/tests/system/amazon/aws/example_dms_serverless.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_dms_serverless_stop_replication]
+ :end-before: [END howto_operator_dms_serverless_stop_replication]
+
+.. _howto/operator:DmsDescribeReplicationsOperator:
+
+Get the status of a serverless replication
+==========================================
+
+To get the status of a serverless replication use
+:class:`~airflow.providers.amazon.aws.operators.dms.DmsDescribeReplicationsOperator`.
+
+.. exampleinclude::
/../../providers/tests/system/amazon/aws/example_dms_serverless.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_dms_serverless_describe_replication]
+ :end-before: [END howto_operator_dms_serverless_describe_replication]
+
+.. _howto/operator:DmsDeleteReplicationConfigOperator:
+
+Delete a serverless replication configuration
+=============================================
+
+To delete a serverless replication config use
+:class:`~airflow.providers.amazon.aws.operators.dms.DmsDeleteReplicationConfigOperator`.
+
+.. exampleinclude::
/../../providers/tests/system/amazon/aws/example_dms_serverless.py
+ :language: python
+ :dedent: 4
+ :start-after: [START
howto_operator_dms_serverless_delete_replication_config]
+ :end-before: [END howto_operator_dms_serverless_delete_replication_config]
+
+
+
Sensors
-------
diff --git a/providers/src/airflow/providers/amazon/aws/hooks/dms.py
b/providers/src/airflow/providers/amazon/aws/hooks/dms.py
index f4bc29cbe9b..e9515c8fac3 100644
--- a/providers/src/airflow/providers/amazon/aws/hooks/dms.py
+++ b/providers/src/airflow/providers/amazon/aws/hooks/dms.py
@@ -18,7 +18,12 @@
from __future__ import annotations
import json
+from datetime import datetime
from enum import Enum
+from typing import Any
+
+from botocore.exceptions import ClientError
+from dateutil import parser
from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
@@ -219,3 +224,159 @@ class DmsHook(AwsBaseHook):
],
WithoutSettings=True,
)
+
+ def describe_replication_configs(self, filters: list[dict] | None = None,
**kwargs) -> list[dict]:
+ """
+ Return list of serverless replication configs.
+
+ .. seealso::
+ -
:external+boto3:py:meth:`DatabaseMigrationService.Client.describe_replication_configs`
+
+ :param filters: List of filter objects
+ :return: List of replication tasks
+ """
+ filters = filters if filters is not None else []
+
+ try:
+ resp = self.conn.describe_replication_configs(Filters=filters,
**kwargs)
+ return resp.get("ReplicationConfigs", [])
+ except Exception as ex:
+ self.log.error("Error while describing replication configs: %s",
str(ex))
+ raise ex
+
+ def create_replication_config(
+ self,
+ replication_config_id: str,
+ source_endpoint_arn: str,
+ target_endpoint_arn: str,
+ compute_config: dict[str, Any],
+ replication_type: str,
+ table_mappings: str,
+ additional_config_kwargs: dict[str, Any] | None = None,
+ **kwargs,
+ ):
+ """
+ Create an AWS DMS Serverless configuration that can be used to start
an DMS Serverless replication.
+
+ .. seealso::
+ -
:external+boto3:py:meth:`DatabaseMigrationService.Client.create_replication_config`
+
+ :param replicationConfigId: Unique identifier used to create a
ReplicationConfigArn.
+ :param sourceEndpointArn: ARN of the source endpoint
+ :param targetEndpointArn: ARN of the target endpoint
+ :param computeConfig: Parameters for provisioning an DMS Serverless
replication.
+ :param replicationType: type of DMS Serverless replication
+ :param tableMappings: JSON table mappings
+ :param tags: Key-value tag pairs
+ :param resourceId: Unique value or name that you set for a given
resource that can be used to construct an Amazon Resource Name (ARN) for that
resource.
+ :param supplementalSettings: JSON settings for specifying supplemental
data
+ :param replicationSettings: JSON settings for DMS Serverless
replications
+
+ :return: ReplicationConfigArn
+
+ """
+ if additional_config_kwargs is None:
+ additional_config_kwargs = {}
+ try:
+ resp = self.conn.create_replication_config(
+ ReplicationConfigIdentifier=replication_config_id,
+ SourceEndpointArn=source_endpoint_arn,
+ TargetEndpointArn=target_endpoint_arn,
+ ComputeConfig=compute_config,
+ ReplicationType=replication_type,
+ TableMappings=table_mappings,
+ **additional_config_kwargs,
+ )
+ arn = resp.get("ReplicationConfig", {}).get("ReplicationConfigArn")
+ self.log.info("Successfully created replication config: %s", arn)
+ return arn
+
+ except ClientError as err:
+ err_str = f"Error: {err.get('Error','').get('Code','')}:
{err.get('Error','').get('Message','')}"
+ self.log.error("Error while creating replication config: %s",
err_str)
+ raise err
+
+ def describe_replications(self, filters: list[dict[str, Any]] | None =
None, **kwargs) -> list[dict]:
+ """
+ Return list of serverless replications.
+
+ .. seealso::
+ -
:external+boto3:py:meth:`DatabaseMigrationService.Client.describe_replications`
+
+ :param filters: List of filter objects
+ :return: List of replications
+ """
+ filters = filters if filters is not None else []
+ try:
+ resp = self.conn.describe_replications(Filters=filters, **kwargs)
+ return resp.get("Replications", [])
+ except Exception as ex:
+ self.log.error("Error while describing replications: %s", str(ex))
+ raise ex
+
+ def delete_replication_config(
+ self, replication_config_arn: str, delay: int = 60, max_attempts: int
= 120
+ ):
+ """
+ Delete an AWS DMS Serverless configuration.
+
+ .. seealso::
+ -
:external+boto3:py:meth:`DatabaseMigrationService.Client.delete_replication_config`
+
+ :param replication_config_arn: ReplicationConfigArn
+ """
+ try:
+ self.log.info("Deleting replication config: %s",
replication_config_arn)
+
+
self.conn.delete_replication_config(ReplicationConfigArn=replication_config_arn)
+
+ except ClientError as err:
+ err_str = (
+ f"Error: {err.get('Error', '').get('Code', '')}:
{err.get('Error', '').get('Message', '')}"
+ )
+ self.log.error("Error while deleting replication config: %s",
err_str)
+ raise err
+
+ def start_replication(
+ self,
+ replication_config_arn: str,
+ start_replication_type: str,
+ cdc_start_time: datetime | str | None = None,
+ cdc_start_pos: str | None = None,
+ cdc_stop_pos: str | None = None,
+ ):
+ additional_args: dict[str, Any] = {}
+
+ if cdc_start_time:
+ additional_args["CdcStartTime"] = (
+ cdc_start_time if isinstance(cdc_start_time, datetime) else
parser.parse(cdc_start_time)
+ )
+ if cdc_start_pos:
+ additional_args["CdcStartPosition"] = cdc_start_pos
+ if cdc_stop_pos:
+ additional_args["CdcStopPosition"] = cdc_stop_pos
+
+ try:
+ resp = self.conn.start_replication(
+ ReplicationConfigArn=replication_config_arn,
+ StartReplicationType=start_replication_type,
+ **additional_args,
+ )
+
+ return resp
+ except Exception as ex:
+ self.log.error("Error while starting replication: %s", str(ex))
+ raise ex
+
+ def stop_replication(self, replication_config_arn: str):
+ resp =
self.conn.stop_replication(ReplicationConfigArn=replication_config_arn)
+ return resp
+
+ def get_provision_status(self, replication_config_arn: str) -> str:
+ """Get the provisioning status for a serverless replication."""
+ result = self.describe_replications(
+ filters=[{"Name": "replication-config-arn", "Values":
[replication_config_arn]}]
+ )
+
+ provision_status = result[0].get("ProvisionData",
{}).get("ProvisionState", "")
+ return provision_status
diff --git a/providers/src/airflow/providers/amazon/aws/operators/dms.py
b/providers/src/airflow/providers/amazon/aws/operators/dms.py
index f4acd3d934d..4e9d118c723 100644
--- a/providers/src/airflow/providers/amazon/aws/operators/dms.py
+++ b/providers/src/airflow/providers/amazon/aws/operators/dms.py
@@ -18,11 +18,22 @@
from __future__ import annotations
from collections.abc import Sequence
-from typing import TYPE_CHECKING, ClassVar
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, ClassVar
+from airflow.configuration import conf
+from airflow.exceptions import AirflowException
from airflow.providers.amazon.aws.hooks.dms import DmsHook
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
+from airflow.providers.amazon.aws.triggers.dms import (
+ DmsReplicationCompleteTrigger,
+ DmsReplicationConfigDeletedTrigger,
+ DmsReplicationDeprovisionedTrigger,
+ DmsReplicationStoppedTrigger,
+ DmsReplicationTerminalStatusTrigger,
+)
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
+from airflow.utils.context import Context
if TYPE_CHECKING:
from airflow.utils.context import Context
@@ -278,3 +289,521 @@ class DmsStopTaskOperator(AwsBaseOperator[DmsHook]):
"""Stop AWS DMS replication task from Airflow."""
self.hook.stop_replication_task(replication_task_arn=self.replication_task_arn)
self.log.info("DMS replication task(%s) is stopping.",
self.replication_task_arn)
+
+
+class DmsDescribeReplicationConfigsOperator(AwsBaseOperator[DmsHook]):
+ """
+ Describes AWS DMS Serverless replication configurations.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the
guide:
+ :ref:`howto/operator:DmsDescribeReplicationConfigsOperator`
+
+ :param describe_config_filter: Filters block for filtering results.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ """
+
+ aws_hook_class = DmsHook
+ template_fields: Sequence[str] = aws_template_fields("filter")
+ template_fields_renderers = {"filter": "json"}
+
+ def __init__(
+ self,
+ *,
+ filter: list[dict] | None = None,
+ aws_conn_id: str | None = "aws_default",
+ **kwargs,
+ ):
+ super().__init__(aws_conn_id=aws_conn_id, **kwargs)
+ self.filter = filter
+
+ def execute(self, context: Context) -> list:
+ """
+ Describe AWS DMS replication configurations.
+
+ :return: List of replication configurations
+ """
+ return self.hook.describe_replication_configs(filters=self.filter)
+
+
+class DmsCreateReplicationConfigOperator(AwsBaseOperator[DmsHook]):
+ """
+ Creates an AWS DMS Serverless replication configuration.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the
guide:
+ :ref:`howto/operator:DmsCreateReplicationConfigOperator`
+
+ :param replication_config_id: Unique identifier used to create a
ReplicationConfigArn.
+ :param source_endpoint_arn: ARN of the source endpoint
+ :param target_endpoint_arn: ARN of the target endpoint
+ :param compute_config: Parameters for provisioning an DMS Serverless
replication.
+ :param replication_type: type of DMS Serverless replication
+ :param table_mappings: JSON table mappings
+ :param tags: Key-value tag pairs
+ :param additional_config_kwargs: Additional configuration parameters for
DMS Serverless replication. Passed directly to the API
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ """
+
+ aws_hook_class = DmsHook
+ template_fields: Sequence[str] = aws_template_fields(
+ "replication_config_id",
+ "source_endpoint_arn",
+ "target_endpoint_arn",
+ "compute_config",
+ "replication_type",
+ "table_mappings",
+ )
+
+ template_fields_renderers = {"compute_config": "json", "tableMappings":
"json"}
+
+ def __init__(
+ self,
+ *,
+ replication_config_id: str,
+ source_endpoint_arn: str,
+ target_endpoint_arn: str,
+ compute_config: dict[str, Any],
+ replication_type: str,
+ table_mappings: str,
+ additional_config_kwargs: dict | None = None,
+ aws_conn_id: str | None = "aws_default",
+ **kwargs,
+ ):
+ super().__init__(
+ aws_conn_id=aws_conn_id,
+ **kwargs,
+ )
+
+ self.replication_config_id = replication_config_id
+ self.source_endpoint_arn = source_endpoint_arn
+ self.target_endpoint_arn = target_endpoint_arn
+ self.compute_config = compute_config
+ self.replication_type = replication_type
+ self.table_mappings = table_mappings
+ self.additional_config_kwargs = additional_config_kwargs or {}
+
+ def execute(self, context: Context) -> str:
+ resp = self.hook.create_replication_config(
+ replication_config_id=self.replication_config_id,
+ source_endpoint_arn=self.source_endpoint_arn,
+ target_endpoint_arn=self.target_endpoint_arn,
+ compute_config=self.compute_config,
+ replication_type=self.replication_type,
+ table_mappings=self.table_mappings,
+ additional_config_kwargs=self.additional_config_kwargs,
+ )
+
+ self.log.info("DMS replication config(%s) has been created.",
self.replication_config_id)
+ return resp
+
+
+class DmsDeleteReplicationConfigOperator(AwsBaseOperator[DmsHook]):
+ """
+ Deletes an AWS DMS Serverless replication configuration.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the
guide:
+ :ref:`howto/operator:DmsDeleteReplicationConfigOperator`
+
+ :param replication_config_arn: ARN of the replication config
+ :param wait_for_completion: If True, waits for the replication config to
be deleted before returning.
+ If False, the operator will return immediately after the request is
made.
+ :param deferrable: Run the operator in deferrable mode.
+ :param waiter_delay: The number of seconds to wait between retries
(default: 60).
+ :param waiter_max_attempts: The maximum number of attempts to be made
(default: 60).
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ """
+
+ aws_hook_class = DmsHook
+ template_fields: Sequence[str] =
aws_template_fields("replication_config_arn")
+
+ VALID_STATES = ["failed", "stopped", "created"]
+ DELETING_STATES = ["deleting"]
+ TERMINAL_PROVISION_STATES = ["deprovisioned", ""]
+
+ def __init__(
+ self,
+ *,
+ replication_config_arn: str,
+ wait_for_completion: bool = True,
+ deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ waiter_delay: int = 60,
+ waiter_max_attempts: int = 60,
+ aws_conn_id: str | None = "aws_default",
+ **kwargs,
+ ):
+ super().__init__(
+ aws_conn_id=aws_conn_id,
+ **kwargs,
+ )
+
+ self.replication_config_arn = replication_config_arn
+ self.wait_for_completion = wait_for_completion
+ self.deferrable = deferrable
+ self.waiter_delay = waiter_delay
+ self.waiter_max_attempts = waiter_max_attempts
+
+ def execute(self, context: Context) -> None:
+ results = self.hook.describe_replications(
+ filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}]
+ )
+
+ current_state = results[0].get("Status", "")
+ self.log.info(
+ "Current state of replication config(%s) is %s.",
self.replication_config_arn, current_state
+ )
+ # replication must be deprovisioned before deleting
+ provision_status =
self.hook.get_provision_status(replication_config_arn=self.replication_config_arn)
+
+ if self.deferrable:
+ if current_state.lower() not in self.VALID_STATES:
+ self.log.info("Deferring until terminal status reached.")
+ self.defer(
+ trigger=DmsReplicationTerminalStatusTrigger(
+ replication_config_arn=self.replication_config_arn,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="retry_execution",
+ )
+ if provision_status not in self.TERMINAL_PROVISION_STATES: # not
deprovisioned:
+ self.log.info("Deferring until deprovisioning completes.")
+ self.defer(
+ trigger=DmsReplicationDeprovisionedTrigger(
+ replication_config_arn=self.replication_config_arn,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="retry_execution",
+ )
+
+ self.hook.get_waiter("replication_terminal_status").wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}],
+ WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts":
self.waiter_max_attempts},
+ )
+ self.hook.get_waiter("replication_deprovisioned").wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}],
+ WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts":
self.waiter_max_attempts},
+ )
+ self.hook.delete_replication_config(self.replication_config_arn)
+ self.handle_delete_wait()
+
+ def handle_delete_wait(self):
+ if self.wait_for_completion:
+ if self.deferrable:
+ self.log.info("Deferring until replication config is deleted.")
+ self.defer(
+ trigger=DmsReplicationConfigDeletedTrigger(
+ replication_config_arn=self.replication_config_arn,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="execute_complete",
+ )
+ else:
+ self.hook.get_waiter("replication_config_deleted").wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}],
+ WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts":
self.waiter_max_attempts},
+ )
+ self.log.info("DMS replication config(%s) deleted.",
self.replication_config_arn)
+
+ def execute_complete(self, context, event=None):
+ self.replication_config_arn = event.get("replication_config_arn")
+ self.log.info("DMS replication config(%s) deleted.",
self.replication_config_arn)
+
+ def retry_execution(self, context, event=None):
+ self.replication_config_arn = event.get("replication_config_arn")
+ self.log.info("Retrying replication config(%s) deletion.",
self.replication_config_arn)
+ self.execute(context)
+
+
+class DmsDescribeReplicationsOperator(AwsBaseOperator[DmsHook]):
+ """
+ Describes AWS DMS Serverless replications.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the
guide:
+ :ref:`howto/operator:DmsDescribeReplicationsOperator`
+
+ :param filter: Filters block for filtering results.
+
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ """
+
+ aws_hook_class = DmsHook
+ template_fields: Sequence[str] = aws_template_fields("filter")
+ template_fields_renderer = {"filter": "json"}
+
+ def __init__(
+ self,
+ *,
+ filter: list[dict[str, Any]] | None = None,
+ aws_conn_id: str | None = "aws_default",
+ **kwargs,
+ ):
+ super().__init__(
+ aws_conn_id=aws_conn_id,
+ **kwargs,
+ )
+
+ self.filter = filter
+
+ def execute(self, context: Context) -> list[dict[str, Any]]:
+ """
+ Describe AWS DMS replications.
+
+ :return: Replications
+ """
+ return self.hook.describe_replications(self.filter)
+
+
+class DmsStartReplicationOperator(AwsBaseOperator[DmsHook]):
+ """
+ Starts an AWS DMS Serverless replication.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the
guide:
+ :ref:`howto/operator:DmsStartReplicationOperator`
+
+ :param replication_config_arn: ARN of the replication config
+ :param replication_start_type: Type of replication.
+ :param cdc_start_time: Start time of CDC
+ :param cdc_start_pos: Indicates when to start CDC.
+ :param cdc_stop_pos: Indicates when to stop CDC.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ """
+
+ RUNNING_STATES = ["running"]
+ STARTABLE_STATES = ["stopped", "failed", "created"]
+ TERMINAL_STATES = ["failed", "stopped", "created"]
+ TERMINAL_PROVISION_STATES = ["deprovisioned", ""]
+
+ aws_hook_class = DmsHook
+ template_fields: Sequence[str] = aws_template_fields(
+ "replication_config_arn", "replication_start_type", "cdc_start_time",
"cdc_start_pos", "cdc_stop_pos"
+ )
+
+ def __init__(
+ self,
+ *,
+ replication_config_arn: str,
+ replication_start_type: str,
+ cdc_start_time: datetime | str | None = None,
+ cdc_start_pos: str | None = None,
+ cdc_stop_pos: str | None = None,
+ wait_for_completion: bool = True,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ aws_conn_id: str | None = "aws_default",
+ **kwargs,
+ ):
+ super().__init__(
+ aws_conn_id=aws_conn_id,
+ **kwargs,
+ )
+
+ self.replication_config_arn = replication_config_arn
+ self.replication_start_type = replication_start_type
+ self.cdc_start_time = cdc_start_time
+ self.cdc_start_pos = cdc_start_pos
+ self.cdc_stop_pos = cdc_stop_pos
+ self.deferrable = deferrable
+ self.waiter_delay = waiter_delay
+ self.waiter_max_attempts = waiter_max_attempts
+ self.wait_for_completion = wait_for_completion
+
+ if self.cdc_start_time and self.cdc_start_pos:
+ raise AirflowException("Only one of cdc_start_time or
cdc_start_pos should be provided.")
+
+ def execute(self, context: Context):
+ result = self.hook.describe_replications(
+ filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}]
+ )
+
+ current_status = result[0].get("Status", "")
+ provision_status =
self.hook.get_provision_status(replication_config_arn=self.replication_config_arn)
+
+ if provision_status == "deprovisioning":
+ # wait for deprovisioning to complete before start/restart
+ self.log.info(
+ "Replication is deprovisioning. Must wait for deprovisioning
before running replication"
+ )
+ if self.deferrable:
+ self.log.info("Deferring until deprovisioning completes.")
+ self.defer(
+ trigger=DmsReplicationDeprovisionedTrigger(
+ replication_config_arn=self.replication_config_arn,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="retry_execution",
+ )
+ else:
+ self.hook.get_waiter("replication_deprovisioned").wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}],
+ WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts":
self.waiter_max_attempts},
+ )
+ provision_status = self.hook.get_provision_status(
+ replication_config_arn=self.replication_config_arn
+ )
+ self.log.info("Replication deprovisioning complete. Provision
status: %s", provision_status)
+
+ if (
+ current_status.lower() in self.STARTABLE_STATES
+ and provision_status in self.TERMINAL_PROVISION_STATES
+ ):
+ resp = self.hook.start_replication(
+ replication_config_arn=self.replication_config_arn,
+ start_replication_type=self.replication_start_type,
+ cdc_start_time=self.cdc_start_time,
+ cdc_start_pos=self.cdc_start_pos,
+ cdc_stop_pos=self.cdc_stop_pos,
+ )
+
+ current_status = resp.get("Replication", {}).get("Status",
"Unknown")
+ self.log.info(
+ "Replication(%s) started with status %s.",
+ self.replication_config_arn,
+ current_status,
+ )
+
+ if self.wait_for_completion:
+ self.log.info("Waiting for %s replication to complete.",
self.replication_config_arn)
+
+ if self.deferrable:
+ self.log.info("Deferring until %s replication completes.",
self.replication_config_arn)
+ self.defer(
+ trigger=DmsReplicationCompleteTrigger(
+ replication_config_arn=self.replication_config_arn,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="execute_complete",
+ )
+
+ self.hook.get_waiter("replication_complete").wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}],
+ WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts":
self.waiter_max_attempts},
+ )
+ self.log.info("Replication(%s) has completed.",
self.replication_config_arn)
+
+ else:
+ self.log.info("Replication(%s) is not in startable state.",
self.replication_config_arn)
+ self.log.info("Status: %s Provision status: %s", current_status,
provision_status)
+
+ def execute_complete(self, context, event=None):
+ self.replication_config_arn = event.get("replication_config_arn")
+ self.log.info("Replication(%s) has completed.",
self.replication_config_arn)
+
+ def retry_execution(self, context, event=None):
+ self.replication_config_arn = event.get("replication_config_arn")
+ self.log.info("Retrying replication %s.", self.replication_config_arn)
+ self.execute(context)
+
+
+class DmsStopReplicationOperator(AwsBaseOperator[DmsHook]):
+ """
+ Stops an AWS DMS Serverless replication.
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the
guide:
+ :ref:`howto/operator:DmsStopReplicationOperator`
+
+ :param replication_config_arn: ARN of the replication config
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ """
+
+ STOPPED_STATES = ["stopped"]
+ NON_STOPPABLE_STATES = ["stopped"]
+
+ aws_hook_class = DmsHook
+ template_fields: Sequence[str] =
aws_template_fields("replication_config_arn")
+
+ def __init__(
+ self,
+ *,
+ replication_config_arn: str,
+ wait_for_completion: bool = True,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ aws_conn_id: str | None = "aws_default",
+ **kwargs,
+ ):
+ super().__init__(
+ aws_conn_id=aws_conn_id,
+ **kwargs,
+ )
+
+ self.replication_config_arn = replication_config_arn
+ self.wait_for_completion = wait_for_completion
+ self.waiter_delay = waiter_delay
+ self.waiter_max_attempts = waiter_max_attempts
+ self.deferrable = deferrable
+
+ def execute(self, context: Context) -> None:
+ results = self.hook.describe_replications(
+ filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}]
+ )
+
+ current_state = results[0].get("Status", "")
+ self.log.info(
+ "Current state of replication config(%s) is %s.",
self.replication_config_arn, current_state
+ )
+
+ if current_state.lower() in self.STOPPED_STATES:
+ self.log.info("DMS replication config(%s) is already stopped.",
self.replication_config_arn)
+ else:
+ resp = self.hook.stop_replication(self.replication_config_arn)
+ status = resp.get("Replication", {}).get("Status", "Unknown")
+ self.log.info(
+ "Stopping DMS replication config(%s). Current status: %s",
self.replication_config_arn, status
+ )
+
+ if self.wait_for_completion:
+ self.log.info("Waiting for %s replication to stop.",
self.replication_config_arn)
+
+ if self.deferrable:
+ self.log.info("Deferring until %s replication stops.",
self.replication_config_arn)
+ self.defer(
+ trigger=DmsReplicationStoppedTrigger(
+ replication_config_arn=self.replication_config_arn,
+ waiter_delay=self.waiter_delay,
+ waiter_max_attempts=self.waiter_max_attempts,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="execute_complete",
+ )
+ self.hook.get_waiter("replication_stopped").wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
[self.replication_config_arn]}],
+ WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts":
self.waiter_max_attempts},
+ )
+
+ def execute_complete(self, context, event=None):
+ self.replication_config_arn = event.get("replication_config_arn")
+ self.log.info("Replication(%s) has stopped.",
self.replication_config_arn)
diff --git a/providers/src/airflow/providers/amazon/aws/triggers/dms.py
b/providers/src/airflow/providers/amazon/aws/triggers/dms.py
new file mode 100644
index 00000000000..99addc96970
--- /dev/null
+++ b/providers/src/airflow/providers/amazon/aws/triggers/dms.py
@@ -0,0 +1,221 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
+from airflow.providers.amazon.aws.hooks.dms import DmsHook
+from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger
+
+if TYPE_CHECKING:
+ from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
+
+
+class DmsReplicationTerminalStatusTrigger(AwsBaseWaiterTrigger):
+ """
+ Trigger when an AWS DMS Serverless replication is in a terminal state.
+
+ :param replication_config_arn: The ARN of the replication config.
+ :param waiter_delay: The amount of time in seconds to wait between
attempts.
+ :param waiter_max_attempts: The maximum number of attempts to be made.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ """
+
+ def __init__(
+ self,
+ replication_config_arn: str,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ aws_conn_id: str | None = "aws_default",
+ ) -> None:
+ super().__init__(
+ serialized_fields={"replication_config_arn":
replication_config_arn},
+ waiter_name="replication_terminal_status",
+ waiter_delay=waiter_delay,
+ waiter_args={"Filters": [{"Name": "replication-config-arn",
"Values": [replication_config_arn]}]},
+ waiter_max_attempts=waiter_max_attempts,
+ failure_message="Replication failed to reach terminal status.",
+ status_message="Status replication is",
+ status_queries=["Replications[0].Status"],
+ return_key="replication_config_arn",
+ return_value=replication_config_arn,
+ aws_conn_id=aws_conn_id,
+ )
+
+ def hook(self) -> AwsGenericHook:
+ return DmsHook(
+ self.aws_conn_id,
+ verify=self.verify,
+ config=self.botocore_config,
+ )
+
+
+class DmsReplicationConfigDeletedTrigger(AwsBaseWaiterTrigger):
+ """
+ Trigger when an AWS DMS Serverless replication config is deleted.
+
+ :param replication_config_arn: The ARN of the replication config.
+ :param waiter_delay: The amount of time in seconds to wait between
attempts.
+ :param waiter_max_attempts: The maximum number of attempts to be made.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ """
+
+ def __init__(
+ self,
+ replication_config_arn: str,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ aws_conn_id: str | None = "aws_default",
+ ) -> None:
+ super().__init__(
+ serialized_fields={"replication_config_arn":
replication_config_arn},
+ waiter_name="replication_config_deleted",
+ waiter_delay=waiter_delay,
+ waiter_args={"Filters": [{"Name": "replication-config-arn",
"Values": [replication_config_arn]}]},
+ waiter_max_attempts=waiter_max_attempts,
+ failure_message="Replication config failed to be deleted.",
+ status_message="Status replication config is",
+ status_queries=["ReplicationConfigs[0].Status"],
+ return_key="replication_config_arn",
+ return_value=replication_config_arn,
+ aws_conn_id=aws_conn_id,
+ )
+
+ def hook(self) -> AwsGenericHook:
+ return DmsHook(
+ self.aws_conn_id,
+ verify=self.verify,
+ config=self.botocore_config,
+ )
+
+
+class DmsReplicationCompleteTrigger(AwsBaseWaiterTrigger):
+ """
+ Trigger when an AWS DMS Serverless replication completes.
+
+ :param replication_config_arn: The ARN of the replication config.
+ :param waiter_delay: The amount of time in seconds to wait between
attempts.
+ :param waiter_max_attempts: The maximum number of attempts to be made.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ """
+
+ def __init__(
+ self,
+ replication_config_arn: str,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ aws_conn_id: str | None = "aws_default",
+ ) -> None:
+ super().__init__(
+ serialized_fields={"replication_config_arn":
replication_config_arn},
+ waiter_name="replication_complete",
+ waiter_delay=waiter_delay,
+ waiter_args={"Filters": [{"Name": "replication-config-arn",
"Values": [replication_config_arn]}]},
+ waiter_max_attempts=waiter_max_attempts,
+ failure_message="Replication failed to complete.",
+ status_message="Status replication is",
+ status_queries=["Replications[0].Status"],
+ return_key="replication_config_arn",
+ return_value=replication_config_arn,
+ aws_conn_id=aws_conn_id,
+ )
+
+ def hook(self) -> AwsGenericHook:
+ return DmsHook(
+ self.aws_conn_id,
+ verify=self.verify,
+ config=self.botocore_config,
+ )
+
+
+class DmsReplicationStoppedTrigger(AwsBaseWaiterTrigger):
+ """
+ Trigger when an AWS DMS Serverless replication is stopped.
+
+ :param replication_config_arn: The ARN of the replication config.
+ :param waiter_delay: The amount of time in seconds to wait between
attempts.
+ :param waiter_max_attempts: The maximum number of attempts to be made.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ """
+
+ def __init__(
+ self,
+ replication_config_arn: str,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ aws_conn_id: str | None = "aws_default",
+ ) -> None:
+ super().__init__(
+ serialized_fields={"replication_config_arn":
replication_config_arn},
+ waiter_name="replication_stopped",
+ waiter_delay=waiter_delay,
+ waiter_args={"Filters": [{"Name": "replication-config-arn",
"Values": [replication_config_arn]}]},
+ waiter_max_attempts=waiter_max_attempts,
+ failure_message="Replication failed to stop.",
+ status_message="Status replication is",
+ status_queries=["Replications[0].Status"],
+ return_key="replication_config_arn",
+ return_value=replication_config_arn,
+ aws_conn_id=aws_conn_id,
+ )
+
+ def hook(self) -> AwsGenericHook:
+ return DmsHook(
+ self.aws_conn_id,
+ verify=self.verify,
+ config=self.botocore_config,
+ )
+
+
+class DmsReplicationDeprovisionedTrigger(AwsBaseWaiterTrigger):
+ """
+ Trigger when an AWS DMS Serverless replication is de-provisioned.
+
+ :param replication_config_arn: The ARN of the replication config.
+ :param waiter_delay: The amount of time in seconds to wait between
attempts.
+ :param waiter_max_attempts: The maximum number of attempts to be made.
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ """
+
+ def __init__(
+ self,
+ replication_config_arn: str,
+ waiter_delay: int = 30,
+ waiter_max_attempts: int = 60,
+ aws_conn_id: str | None = "aws_default",
+ ) -> None:
+ super().__init__(
+ serialized_fields={"replication_config_arn":
replication_config_arn},
+ waiter_name="replication_deprovisioned",
+ waiter_delay=waiter_delay,
+ waiter_args={"Filters": [{"Name": "replication-config-arn",
"Values": [replication_config_arn]}]},
+ waiter_max_attempts=waiter_max_attempts,
+ failure_message="Replication failed to deprovision.",
+ status_message="Status replication is",
+ status_queries=["Replications[0].ProvisionData.ProvisionState"],
+ return_key="replication_config_arn",
+ return_value=replication_config_arn,
+ aws_conn_id=aws_conn_id,
+ )
+
+ def hook(self) -> AwsGenericHook:
+ return DmsHook(
+ self.aws_conn_id,
+ verify=self.verify,
+ config=self.botocore_config,
+ )
diff --git a/providers/src/airflow/providers/amazon/aws/waiters/dms.json
b/providers/src/airflow/providers/amazon/aws/waiters/dms.json
new file mode 100644
index 00000000000..4fab7825ae1
--- /dev/null
+++ b/providers/src/airflow/providers/amazon/aws/waiters/dms.json
@@ -0,0 +1,88 @@
+{
+ "version": 2,
+ "waiters": {
+ "replication_deprovisioned": {
+ "operation": "DescribeReplications",
+ "delay": 30,
+ "maxAttempts": 60,
+ "acceptors": [
+ {
+ "matcher": "path",
+ "argument": "Replications[0].ProvisionData.ProvisionState",
+ "expected": "deprovisioned",
+ "state": "success"
+ }
+ ]
+ },
+ "replication_terminal_status": {
+ "operation": "DescribeReplications",
+ "delay": 30,
+ "maxAttempts": 60,
+ "acceptors": [
+ {
+ "matcher": "path",
+ "argument": "Replications[0].ProvisionData.ProvisionState",
+ "expected": "deprovisioning",
+ "state": "retry"
+ },
+ {
+ "matcher": "path",
+ "argument": "Replications[0].Status",
+ "expected": "failed",
+ "state": "success"
+ },
+ {
+ "matcher": "path",
+ "argument": "Replications[0].Status",
+ "expected": "stopped",
+ "state": "success"
+ }
+ ]
+ },
+ "replication_complete": {
+ "operation": "DescribeReplications",
+ "delay": 30,
+ "maxAttempts": 60,
+ "acceptors": [
+ {
+ "matcher": "path",
+ "argument": "Replications[0].Status",
+ "expected": "failed",
+ "state": "failure"
+ },
+ {
+ "matcher": "path",
+ "argument": "Replications[0].Status",
+ "expected": "stopped",
+ "state": "success"
+ }
+ ]
+ },
+ "replication_config_deleted": {
+ "operation": "DescribeReplicationConfigs",
+ "delay": 5,
+ "maxAttempts": 60,
+ "acceptors": [
+ {
+ "matcher": "error",
+ "expected": "ResourceNotFoundFault",
+ "state": "success",
+ "argument": "Error.Code"
+ }
+ ]
+ },
+ "replication_stopped": {
+ "operation": "DescribeReplications",
+ "delay": 5,
+ "maxAttempts": 60,
+ "acceptors": [
+ {
+ "matcher": "path",
+ "argument": "Replications[0].Status",
+ "expected": "stopped",
+ "state": "success"
+ }
+ ]
+ }
+ }
+}
diff --git a/providers/src/airflow/providers/amazon/provider.yaml
b/providers/src/airflow/providers/amazon/provider.yaml
index 493d5b15928..f698ffdd4f3 100644
--- a/providers/src/airflow/providers/amazon/provider.yaml
+++ b/providers/src/airflow/providers/amazon/provider.yaml
@@ -778,6 +778,10 @@ triggers:
- integration-name: Amazon Neptune
python-modules:
- airflow.providers.amazon.aws.triggers.neptune
+ - integration-name: AWS Database Migration Service
+ python-modules:
+ - airflow.providers.amazon.aws.triggers.dms
+
transfers:
- source-integration-name: Amazon DynamoDB
diff --git a/providers/tests/amazon/aws/hooks/test_dms.py
b/providers/tests/amazon/aws/hooks/test_dms.py
index 9d66df55c25..ef7095d54a1 100644
--- a/providers/tests/amazon/aws/hooks/test_dms.py
+++ b/providers/tests/amazon/aws/hooks/test_dms.py
@@ -17,10 +17,12 @@
from __future__ import annotations
import json
+from datetime import datetime
from typing import Any
from unittest import mock
import pytest
+from dateutil import parser
from airflow.providers.amazon.aws.hooks.dms import DmsHook, DmsTaskWaiterStatus
@@ -66,6 +68,187 @@ MOCK_START_RESPONSE: dict[str, Any] = {"ReplicationTask":
{**MOCK_TASK_RESPONSE_
MOCK_STOP_RESPONSE: dict[str, Any] = {"ReplicationTask":
{**MOCK_TASK_RESPONSE_DATA, "Status": "stopping"}}
MOCK_DELETE_RESPONSE: dict[str, Any] = {"ReplicationTask":
{**MOCK_TASK_RESPONSE_DATA, "Status": "deleting"}}
+MOCK_CONFIG_RESPONSE: dict[str, Any] = {
+ "Marker": "xxxxx",
+ "ReplicationConfigs": [
+ {
+ "ReplicationConfigIdentifier": "1111",
+ "ReplicationConfigArn": "arn:aws:my-arn",
+ "SourceEndpointArn": "source-endpoint-arn",
+ "TargetEndpointArn": "target-endpoint-arn",
+ "ReplicationType": "cdc",
+ "ComputeConfig": {
+ "AvailabilityZone": "az1",
+ "MaxCapacityUnits": 10,
+ "MinCapacityUnits": 20,
+ "MultiAZ": True,
+ "PreferredMaintenanceWindow": "string",
+ "ReplicationSubnetGroupId": "string",
+ "VpcSecurityGroupIds": [
+ "string",
+ ],
+ },
+ "ReplicationSettings": "string",
+ "SupplementalSettings": "string",
+ "TableMappings": "string",
+ "ReplicationConfigCreateTime": datetime(2015, 1, 1),
+ "ReplicationConfigUpdateTime": datetime(2015, 1, 1),
+ },
+ {
+ "ReplicationConfigIdentifier": "2222",
+ "ReplicationConfigArn": "arn:aws:my-arn",
+ "SourceEndpointArn": "source-endpoint-arn",
+ "TargetEndpointArn": "target-endpoint-arn",
+ "ReplicationType": "full-load-and-cdc",
+ "ComputeConfig": {
+ "AvailabilityZone": "string",
+ "DnsNameServers": "string",
+ "KmsKeyId": "string",
+ "MaxCapacityUnits": 1,
+ "MinCapacityUnits": 30,
+ "MultiAZ": False,
+ "PreferredMaintenanceWindow": "string",
+ "ReplicationSubnetGroupId": "string",
+ "VpcSecurityGroupIds": [
+ "string",
+ ],
+ },
+ "ReplicationSettings": "string",
+ "SupplementalSettings": "string",
+ "TableMappings": "string",
+ "ReplicationConfigCreateTime": datetime(2015, 1, 1),
+ "ReplicationConfigUpdateTime": datetime(2015, 2, 1),
+ },
+ ],
+}
+
+MOCK_REPLICATION_CONFIG: dict[str, Any] = {
+ "ReplicationConfigIdentifier": "test-config",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ComputeConfig": {
+ "MaxCapacityUnits": 2,
+ "MinCapacityUnits": 4,
+ },
+ "ReplicationType": "full-load",
+ "TableMappings": json.dumps(
+ {
+ "TableMappings": [
+ {
+ "Type": "Selection",
+ "RuleId": 123,
+ "RuleName": "test-rule",
+ "SourceSchema": "/",
+ "SourceTable": "/",
+ }
+ ]
+ }
+ ),
+ "ReplicationSettings": "string",
+ "SupplementalSettings": "string",
+ "ResourceIdentifier": "string",
+}
+
+MOCK_REPLICATION_CONFIG_RESP: dict[str, Any] = {
+ "ReplicationConfig": {
+ "ReplicationConfigIdentifier": "test-config",
+ "ReplicationConfigArn":
"arn:aws:dms:us-east-1:123456789012:replication-config/test-config",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ReplicationType": "full-load",
+ }
+}
+
+MOCK_DESCRIBE_REPLICATIONS_RESP = {
+ "Marker": "string",
+ "Replications": [
+ {
+ "ReplicationConfigIdentifier": "test-config",
+ "ReplicationConfigArn":
"arn:aws:dms:us-east-1:123456789012:replication-config/test-config",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ReplicationType": "full-load",
+ "Status": "CREATED",
+ "ProvisionData": {
+ "ProvisionState": "string",
+ "ProvisionedCapacityUnits": 123,
+ "DateProvisioned": datetime(2015, 1, 1),
+ "IsNewProvisioningAvailable": False,
+ "DateNewProvisioningDataAvailable": datetime(2015, 1, 1),
+ "ReasonForNewProvisioningData": "string",
+ },
+ "StopReason": "string",
+ "FailureMessages": [
+ "string",
+ ],
+ "ReplicationStats": {
+ "FullLoadProgressPercent": 123,
+ "ElapsedTimeMillis": 123,
+ "TablesLoaded": 123,
+ "TablesLoading": 123,
+ "TablesQueued": 123,
+ "TablesErrored": 123,
+ "FreshStartDate": datetime(2015, 1, 1),
+ "StartDate": datetime(2015, 1, 1),
+ "StopDate": datetime(2015, 1, 1),
+ "FullLoadStartDate": datetime(2015, 1, 1),
+ "FullLoadFinishDate": datetime(2015, 1, 1),
+ },
+ "StartReplicationType": "string",
+ "CdcStartTime": datetime(2015, 1, 1),
+ "CdcStartPosition": "string",
+ "CdcStopPosition": "string",
+ "RecoveryCheckpoint": "string",
+ "ReplicationCreateTime": datetime(2015, 1, 1),
+ "ReplicationUpdateTime": datetime(2015, 1, 1),
+ "ReplicationLastStopTime": datetime(2015, 1, 1),
+ "ReplicationDeprovisionTime": datetime(2015, 1, 1),
+ },
+ {
+ "ReplicationConfigIdentifier": "test-config-2",
+ "ReplicationConfigArn":
"arn:aws:dms:us-east-1:123456789012:replication-config/test-config-2",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ReplicationType": "cdc-only",
+ "Status": "CREATED",
+ "ProvisionData": {
+ "ProvisionState": "string",
+ "ProvisionedCapacityUnits": 123,
+ "DateProvisioned": datetime(2015, 1, 1),
+ "IsNewProvisioningAvailable": False,
+ "DateNewProvisioningDataAvailable": datetime(2015, 1, 1),
+ "ReasonForNewProvisioningData": "string",
+ },
+ "StopReason": "string",
+ "FailureMessages": [
+ "string",
+ ],
+ "ReplicationStats": {
+ "FullLoadProgressPercent": 123,
+ "ElapsedTimeMillis": 123,
+ "TablesLoaded": 123,
+ "TablesLoading": 123,
+ "TablesQueued": 123,
+ "TablesErrored": 123,
+ "FreshStartDate": datetime(2015, 1, 1),
+ "StartDate": datetime(2015, 1, 1),
+ "StopDate": datetime(2015, 1, 1),
+ "FullLoadStartDate": datetime(2015, 1, 1),
+ "FullLoadFinishDate": datetime(2015, 1, 1),
+ },
+ "StartReplicationType": "string",
+ "CdcStartTime": datetime(2015, 1, 1),
+ "CdcStartPosition": "string",
+ "CdcStopPosition": "string",
+ "RecoveryCheckpoint": "string",
+ "ReplicationCreateTime": datetime(2015, 1, 1),
+ "ReplicationUpdateTime": datetime(2015, 1, 1),
+ "ReplicationLastStopTime": datetime(2015, 1, 1),
+ "ReplicationDeprovisionTime": datetime(2015, 1, 1),
+ },
+ ],
+}
+
class TestDmsHook:
def setup_method(self):
@@ -213,3 +396,117 @@ class TestDmsHook:
}
mock_conn.return_value.get_waiter.assert_called_with("replication_task_deleted")
mock_conn.return_value.get_waiter.return_value.wait.assert_called_with(**expected_waiter_call_params)
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_describe_config_no_filter(self, mock_conn):
+ mock_conn.describe_replication_configs.return_value =
MOCK_CONFIG_RESPONSE
+ resp = self.dms.describe_replication_configs()
+
+ assert len(resp) == 2
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_describe_config_filter(self, mock_conn):
+ filter = [{"Name": "replication-type", "Values": ["cdc"]}]
+ self.dms.describe_replication_configs(filters=filter)
+
mock_conn.describe_replication_configs.assert_called_with(Filters=filter)
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_create_repl_config_kwargs(self, mock_conn):
+ self.dms.create_replication_config(
+
replication_config_id=MOCK_REPLICATION_CONFIG["ReplicationConfigIdentifier"],
+ source_endpoint_arn=MOCK_REPLICATION_CONFIG["SourceEndpointArn"],
+ target_endpoint_arn=MOCK_REPLICATION_CONFIG["TargetEndpointArn"],
+ compute_config=MOCK_REPLICATION_CONFIG["ComputeConfig"],
+ replication_type=MOCK_REPLICATION_CONFIG["ReplicationType"],
+ table_mappings=MOCK_REPLICATION_CONFIG["TableMappings"],
+ additional_config_kwargs={
+ "ReplicationSettings":
MOCK_REPLICATION_CONFIG["ReplicationSettings"],
+ "SupplementalSettings":
MOCK_REPLICATION_CONFIG["SupplementalSettings"],
+ },
+ )
+
+ mock_conn.create_replication_config.assert_called_with(
+
ReplicationConfigIdentifier=MOCK_REPLICATION_CONFIG["ReplicationConfigIdentifier"],
+ SourceEndpointArn=MOCK_REPLICATION_CONFIG["SourceEndpointArn"],
+ TargetEndpointArn=MOCK_REPLICATION_CONFIG["TargetEndpointArn"],
+ ComputeConfig=MOCK_REPLICATION_CONFIG["ComputeConfig"],
+ ReplicationType=MOCK_REPLICATION_CONFIG["ReplicationType"],
+ TableMappings=MOCK_REPLICATION_CONFIG["TableMappings"],
+ ReplicationSettings=MOCK_REPLICATION_CONFIG["ReplicationSettings"],
+
SupplementalSettings=MOCK_REPLICATION_CONFIG["SupplementalSettings"],
+ )
+
+ self.dms.create_replication_config(
+
replication_config_id=MOCK_REPLICATION_CONFIG["ReplicationConfigIdentifier"],
+ source_endpoint_arn=MOCK_REPLICATION_CONFIG["SourceEndpointArn"],
+ target_endpoint_arn=MOCK_REPLICATION_CONFIG["TargetEndpointArn"],
+ compute_config=MOCK_REPLICATION_CONFIG["ComputeConfig"],
+ replication_type=MOCK_REPLICATION_CONFIG["ReplicationType"],
+ table_mappings=MOCK_REPLICATION_CONFIG["TableMappings"],
+ )
+ mock_conn.create_replication_config.assert_called_with(
+
ReplicationConfigIdentifier=MOCK_REPLICATION_CONFIG["ReplicationConfigIdentifier"],
+ SourceEndpointArn=MOCK_REPLICATION_CONFIG["SourceEndpointArn"],
+ TargetEndpointArn=MOCK_REPLICATION_CONFIG["TargetEndpointArn"],
+ ComputeConfig=MOCK_REPLICATION_CONFIG["ComputeConfig"],
+ ReplicationType=MOCK_REPLICATION_CONFIG["ReplicationType"],
+ TableMappings=MOCK_REPLICATION_CONFIG["TableMappings"],
+ )
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_create_repl_config(self, mock_conn):
+ mock_conn.create_replication_config.return_value =
MOCK_REPLICATION_CONFIG_RESP
+
+ resp = self.dms.create_replication_config(
+
replication_config_id=MOCK_REPLICATION_CONFIG["ReplicationConfigIdentifier"],
+ source_endpoint_arn=MOCK_REPLICATION_CONFIG["SourceEndpointArn"],
+ target_endpoint_arn=MOCK_REPLICATION_CONFIG["TargetEndpointArn"],
+ compute_config=MOCK_REPLICATION_CONFIG["ComputeConfig"],
+ replication_type=MOCK_REPLICATION_CONFIG["ReplicationType"],
+ table_mappings=MOCK_REPLICATION_CONFIG["TableMappings"],
+ )
+
+ assert resp ==
MOCK_REPLICATION_CONFIG_RESP["ReplicationConfig"]["ReplicationConfigArn"]
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_describe_replications(self, mock_conn):
+ mock_conn.describe_replication_tasks.return_value =
MOCK_DESCRIBE_REPLICATIONS_RESP
+ resp = self.dms.describe_replication_tasks()
+ assert len(resp) == 2
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_describe_replications_filter(self, mock_conn):
+ filter = [
+ {
+ "Name": "replication-task-id",
+ "Values":
MOCK_DESCRIBE_REPLICATIONS_RESP["Replications"][0]["ReplicationConfigArn"],
+ }
+ ]
+ self.dms.describe_replication_tasks(filters=filter)
+ mock_conn.describe_replication_tasks.assert_called_with(filters=filter)
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_start_replication_args(self, mock_conn):
+ self.dms.start_replication(
+ replication_config_arn=MOCK_TASK_ARN,
+ start_replication_type="cdc",
+ )
+ mock_conn.start_replication.assert_called_with(
+ ReplicationConfigArn=MOCK_TASK_ARN,
+ StartReplicationType="cdc",
+ )
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_start_replication_kwargs(self, mock_conn):
+ self.dms.start_replication(
+ replication_config_arn=MOCK_TASK_ARN,
+ start_replication_type="cdc",
+ cdc_start_time="2022-01-01T00:00:00Z",
+ cdc_start_pos=None,
+ cdc_stop_pos=None,
+ )
+ mock_conn.start_replication.assert_called_with(
+ ReplicationConfigArn=MOCK_TASK_ARN,
+ StartReplicationType="cdc",
+ CdcStartTime=parser.parse("2022-01-01T00:00:00Z"),
+ )
diff --git a/providers/tests/amazon/aws/operators/test_dms.py
b/providers/tests/amazon/aws/operators/test_dms.py
index 9a522c525cb..495beed5446 100644
--- a/providers/tests/amazon/aws/operators/test_dms.py
+++ b/providers/tests/amazon/aws/operators/test_dms.py
@@ -17,21 +17,33 @@
from __future__ import annotations
import json
+from typing import Any
from unittest import mock
import pendulum
import pytest
-from airflow import DAG
-from airflow.models import DagRun, TaskInstance
+from airflow.exceptions import AirflowException, TaskDeferred
+from airflow.models import DAG, DagRun, TaskInstance
+from airflow.models.variable import Variable
from airflow.providers.amazon.aws.hooks.dms import DmsHook
from airflow.providers.amazon.aws.operators.dms import (
+ DmsCreateReplicationConfigOperator,
DmsCreateTaskOperator,
+ DmsDeleteReplicationConfigOperator,
DmsDeleteTaskOperator,
+ DmsDescribeReplicationConfigsOperator,
+ DmsDescribeReplicationsOperator,
DmsDescribeTasksOperator,
+ DmsStartReplicationOperator,
DmsStartTaskOperator,
+ DmsStopReplicationOperator,
DmsStopTaskOperator,
)
+from airflow.providers.amazon.aws.triggers.dms import (
+ DmsReplicationDeprovisionedTrigger,
+ DmsReplicationTerminalStatusTrigger,
+)
from airflow.utils import timezone
from airflow.utils.types import DagRunType
@@ -450,3 +462,579 @@ class TestDmsStopTaskOperator:
)
validate_template_fields(op)
+
+
+class TestDmsDescribeReplicationConfigsOperator:
+ filter = [{"Name": "replication-type", "Values": ["cdc"]}]
+
+ def test_init(self):
+ op = DmsDescribeReplicationConfigsOperator(task_id="test_task")
+ assert op.filter is None
+
+ @pytest.mark.db_test
+ @mock.patch.object(DmsHook, "conn")
+ def test_template_fields_native(self, mock_conn, session):
+ execution_date = timezone.datetime(2020, 1, 1)
+ Variable.set("test_filter", self.filter, session=session)
+
+ dag = DAG(
+ "test_dms",
+ schedule=None,
+ start_date=execution_date,
+ render_template_as_native_obj=True,
+ )
+ op = DmsDescribeReplicationConfigsOperator(
+ task_id="test_task", filter="{{ var.value.test_filter }}", dag=dag
+ )
+
+ dag_run = DagRun(
+ dag_id=dag.dag_id,
+ run_id="test",
+ run_type=DagRunType.MANUAL,
+ )
+ ti = TaskInstance(task=op)
+ ti.dag_run = dag_run
+ session.add(ti)
+ session.commit()
+ context = ti.get_template_context(session)
+ ti.render_templates(context)
+
+ assert op.filter == self.filter
+
+
+class TestDmsCreateReplicationConfigOperator:
+ TASK_DATA = {
+ "ReplicationConfigIdentifier": "test-config",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ComputeConfig": {
+ "MaxCapacityUnits": 2,
+ "MinCapacityUnits": 4,
+ },
+ "ReplicationType": "full-load",
+ "TableMappings": json.dumps(
+ {
+ "TableMappings": [
+ {
+ "Type": "Selection",
+ "RuleId": 123,
+ "RuleName": "test-rule",
+ "SourceSchema": "/",
+ "SourceTable": "/",
+ }
+ ]
+ }
+ ),
+ "ReplicationSettings": "string",
+ "SupplementalSettings": "string",
+ "ResourceIdentifier": "string",
+ }
+
+ MOCK_REPLICATION_CONFIG_RESP: dict[str, Any] = {
+ "ReplicationConfig": {
+ "ReplicationConfigIdentifier": "test-config",
+ "ReplicationConfigArn":
"arn:aws:dms:us-east-1:123456789012:replication-config/test-config",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ReplicationType": "full-load",
+ }
+ }
+
+ def test_init(self):
+ DmsCreateReplicationConfigOperator(
+ task_id="create_replication_config",
+
replication_config_id=self.TASK_DATA["ReplicationConfigIdentifier"],
+ source_endpoint_arn=self.TASK_DATA["SourceEndpointArn"],
+ target_endpoint_arn=self.TASK_DATA["TargetEndpointArn"],
+ replication_type=self.TASK_DATA["ReplicationType"],
+ table_mappings=self.TASK_DATA["TableMappings"],
+ compute_config=self.TASK_DATA["ComputeConfig"],
+ )
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_operator(self, mock_hook):
+ mock_hook.create_replication_config.return_value =
self.MOCK_REPLICATION_CONFIG_RESP
+ op = DmsCreateReplicationConfigOperator(
+ task_id="create_replication_config",
+
replication_config_id=self.TASK_DATA["ReplicationConfigIdentifier"],
+ source_endpoint_arn=self.TASK_DATA["SourceEndpointArn"],
+ target_endpoint_arn=self.TASK_DATA["TargetEndpointArn"],
+ replication_type=self.TASK_DATA["ReplicationType"],
+ table_mappings=self.TASK_DATA["TableMappings"],
+ compute_config=self.TASK_DATA["ComputeConfig"],
+ )
+ resp = op.execute(None)
+ assert resp ==
self.MOCK_REPLICATION_CONFIG_RESP["ReplicationConfig"]["ReplicationConfigArn"]
+
+
+class TestDmsDeleteReplicationConfigOperator:
+ TASK_DATA = {
+ "ReplicationConfigIdentifier": "test-config",
+ "ReplicationConfigArn": "arn:xxxxxx",
+ "SourceEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:RZZK4EZW5UANC7Y3P4E776WHBE",
+ "TargetEndpointArn":
"arn:aws:dms:us-east-1:123456789012:endpoint:GVBUJQXJZASXWHTWCLN2WNT57E",
+ "ComputeConfig": {
+ "MaxCapacityUnits": 2,
+ "MinCapacityUnits": 4,
+ },
+ "ReplicationType": "full-load",
+ "TableMappings": json.dumps(
+ {
+ "TableMappings": [
+ {
+ "Type": "Selection",
+ "RuleId": 123,
+ "RuleName": "test-rule",
+ "SourceSchema": "/",
+ "SourceTable": "/",
+ }
+ ]
+ }
+ ),
+ "ReplicationSettings": "string",
+ "SupplementalSettings": "string",
+ "ResourceIdentifier": "string",
+ }
+
+ def get_replication_status(self, status: str, deprovisioned: str =
"deprovisioned"):
+ return [
+ {
+ "Status": status,
+ "ReplicationArn": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+ "ReplicationIdentifier": "test-config",
+ "SourceEndpointArn": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+ "TargetEndpointArn": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+ "ProvisionData": {"ProvisionState": deprovisioned,
"ProvisionedCapacityUnits": 2},
+ }
+ ]
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsDeleteReplicationConfigOperator,
"handle_delete_wait")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_happy_path(self, mock_waiter, mock_handle,
mock_describe_replications, mock_conn):
+ # testing all good statuses and no waiting
+ mock_describe_replications.return_value = self.get_replication_status(
+ status="stopped", deprovisioned="deprovisioned"
+ )
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=False,
+ wait_for_completion=False,
+ )
+ op.execute({})
+
+ mock_conn.delete_replication_config.assert_called_once()
+ mock_waiter.assert_has_calls(
+ [
+ mock.call("replication_terminal_status"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ mock.call("replication_deprovisioned"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ ]
+ )
+ mock_handle.assert_called_once()
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ def test_defer_not_ready(self, mock_describe, mock_conn):
+ mock_describe.return_value = self.get_replication_status("running")
+
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=True,
+ )
+
+ with pytest.raises(TaskDeferred) as defer:
+ op.execute({})
+
+ assert isinstance(defer.value.trigger,
DmsReplicationTerminalStatusTrigger)
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_wait_for_completion(self, mock_waiter,
mock_describe_replications, mock_conn):
+ mock_describe_replications.return_value = self.get_replication_status(
+ status="failed", deprovisioned="deprovisioned"
+ )
+
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=False,
+ wait_for_completion=True,
+ )
+ op.execute({})
+
+ mock_waiter.assert_has_calls(
+ [
+ mock.call("replication_terminal_status"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ mock.call("replication_deprovisioned"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ ]
+ )
+
+ # mock_waiter.assert_called_with("replication_config_deleted")
+ # mock_waiter.assert_called_once()
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_wait_for_completion_not_ready(self, mock_waiter,
mock_describe_replications, mock_conn):
+ mock_describe_replications.return_value = self.get_replication_status(
+ status="failed", deprovisioned="xxx"
+ )
+
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=False,
+ wait_for_completion=True,
+ )
+ op.execute({})
+
+ mock_waiter.assert_has_calls(
+ [
+ mock.call("replication_deprovisioned"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ mock.call("replication_config_deleted"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ ]
+ )
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsDeleteReplicationConfigOperator,
"handle_delete_wait")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_not_ready_state(self, mock_waiter, mock_handle, mock_describe,
mock_conn):
+ mock_describe.return_value = self.get_replication_status("running")
+
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=False,
+ wait_for_completion=False,
+ )
+ op.execute({})
+
+ mock_waiter.assert_has_calls(
+ [
+ mock.call("replication_terminal_status"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ mock.call("replication_deprovisioned"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ ]
+ )
+ mock_handle.assert_called_once()
+ mock_conn.delete_replication_config.assert_called_once()
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsDeleteReplicationConfigOperator,
"handle_delete_wait")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_not_deprovisioned(self, mock_waiter, mock_handle, mock_describe,
mock_conn):
+ mock_describe.return_value = self.get_replication_status("stopped",
"deprovisioning")
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=False,
+ wait_for_completion=False,
+ )
+ op.execute({})
+
+ mock_waiter.assert_has_calls(
+ [
+ mock.call("replication_terminal_status"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ mock.call("replication_deprovisioned"),
+ mock.call().wait(
+ Filters=[{"Name": "replication-config-arn", "Values":
["arn:xxxxxx"]}],
+ WaiterConfig={"Delay": 60, "MaxAttempts": 60},
+ ),
+ ]
+ )
+ mock_handle.assert_called_once()
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_config_not_found(self, mock_waiter, mock_describe, mock_conn):
+ mock_describe.return_value = []
+
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=False,
+ wait_for_completion=False,
+ )
+ with pytest.raises(IndexError):
+ op.execute({})
+ mock_waiter.assert_not_called()
+ mock_conn.delete_replication_config.assert_not_called()
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_defer_not_deprovisioned(self, mock_waiter, mock_describe,
mock_conn):
+ # not deprovisioned
+ mock_describe.return_value = self.get_replication_status("stopped",
"deprovisioning")
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=True,
+ wait_for_completion=False,
+ )
+
+ with pytest.raises(TaskDeferred) as defer:
+ op.execute({})
+
+ assert isinstance(defer.value.trigger,
DmsReplicationDeprovisionedTrigger)
+
+ # not in terminal status
+ mock_describe.return_value = self.get_replication_status("running",
"deprovisioning")
+ op = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ replication_config_arn=self.TASK_DATA["ReplicationConfigArn"],
+ deferrable=True,
+ wait_for_completion=False,
+ )
+
+ with pytest.raises(TaskDeferred) as defer:
+ op.execute({})
+
+ assert isinstance(defer.value.trigger,
DmsReplicationTerminalStatusTrigger)
+
+
+class TestDmsDescribeReplicationsOperator:
+ FILTER = [{"Name": "replication-type", "Values": ["cdc"]}]
+
+ def get_replications(self):
+ return {
+ "Replications": [
+ {
+ "Status": "test",
+ "ReplicationArn": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+ "ReplicationIdentifier": "test-config",
+ "SourceEndpointArn": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+ "TargetEndpointArn": "XXXXXXXXXXXXXXXXXXXXXXXXX",
+ }
+ ]
+ }
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_filter(self, mock_conn):
+ mock_conn.describe_replications.return_value = self.get_replications()
+
+ op = DmsDescribeReplicationsOperator(
+ task_id="test_task",
+ filter=self.FILTER,
+ )
+
+ res = op.execute({})
+
+
mock_conn.describe_replications.assert_called_once_with(Filters=self.FILTER)
+ assert isinstance(res, list)
+
+ @mock.patch.object(DmsHook, "conn")
+ def test_filter_none(self, mock_conn):
+ mock_conn.describe_replications.return_value = self.get_replications()
+
+ op = DmsDescribeReplicationsOperator(
+ task_id="test_task",
+ )
+
+ res = op.execute({})
+
+ mock_conn.describe_replications.assert_called_once_with(Filters=[])
+ assert isinstance(res, list)
+
+
+class TestDmsStartReplicationOperator:
+ def mock_describe_replication_response(self, status: str):
+ return [
+ {
+ "ReplicationConfigIdentifier": "string",
+ "ReplicationConfigArn": "string",
+ "SourceEndpointArn": "string",
+ "TargetEndpointArn": "string",
+ "ReplicationType": "full-load",
+ "Status": status,
+ }
+ ]
+
+ def mock_replication_response(self, status: str):
+ return {
+ "Replication": {
+ "ReplicationConfigIdentifier": "xxxx",
+ "ReplicationConfigArn": "xxxx",
+ "Status": status,
+ }
+ }
+
+ def test_arg_validation(self):
+ with pytest.raises(AirflowException):
+ DmsStartReplicationOperator(
+ task_id="start_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ replication_start_type="cdc",
+ cdc_start_pos=1,
+ cdc_start_time="2024-01-01 00:00:00",
+ )
+ DmsStartReplicationOperator(
+ task_id="start_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ replication_start_type="cdc",
+ cdc_start_pos=1,
+ )
+
+ DmsStartReplicationOperator(
+ task_id="start_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ replication_start_type="cdc",
+ cdc_start_time="2024-01-01 00:00:00",
+ )
+
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "start_replication")
+ def test_already_running(self, mock_replication, mock_describe):
+ mock_describe.return_value =
self.mock_describe_replication_response("test")
+
+ op = DmsStartReplicationOperator(
+ task_id="start_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ replication_start_type="cdc",
+ cdc_start_pos=1,
+ wait_for_completion=False,
+ deferrable=False,
+ )
+
+ op.execute({})
+ assert mock_replication.call_count == 0
+
+ mock_describe.return_value =
self.mock_describe_replication_response("failed")
+ op.execute({})
+ mock_replication.return_value =
self.mock_replication_response("running")
+ assert mock_replication.call_count == 1
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "get_waiter")
+ @mock.patch.object(DmsHook, "describe_replications")
+ def test_wait_for_completion(self, mock_describe, mock_waiter, mock_conn):
+ mock_describe.return_value =
self.mock_describe_replication_response("stopped")
+ op = DmsStartReplicationOperator(
+ task_id="start_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ replication_start_type="cdc",
+ cdc_start_pos=1,
+ wait_for_completion=True,
+ deferrable=False,
+ )
+
+ op.execute({})
+ mock_waiter.assert_called_with("replication_complete")
+ mock_waiter.assert_called_once()
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ def test_execute(self, mock_describe, mock_conn):
+ mock_describe.return_value =
self.mock_describe_replication_response("stopped")
+
+ op = DmsStartReplicationOperator(
+ task_id="start_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ replication_start_type="cdc",
+ cdc_start_pos=1,
+ wait_for_completion=False,
+ deferrable=False,
+ )
+ op.execute({})
+ assert mock_conn.start_replication.call_count == 1
+
+
+class TestDmsStopReplicationOperator:
+ def mock_describe_replication_response(self, status: str):
+ return [
+ {
+ "ReplicationConfigIdentifier": "string",
+ "ReplicationConfigArn": "string",
+ "SourceEndpointArn": "string",
+ "TargetEndpointArn": "string",
+ "ReplicationType": "full-load",
+ "Status": status,
+ }
+ ]
+
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "conn")
+ def test_already_stopped(self, mock_conn, mock_describe_replications):
+ mock_describe_replications.return_value =
self.mock_describe_replication_response("stopped")
+
+ op = DmsStopReplicationOperator(
+ task_id="stop_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ wait_for_completion=False,
+ deferrable=False,
+ )
+ op.execute({})
+
+ assert mock_conn.stop_replication.call_count == 0
+
+ @mock.patch.object(DmsHook, "stop_replication")
+ @mock.patch.object(DmsHook, "describe_replications")
+ def test_execute(self, mock_describe_replications, mock_stop):
+ mock_describe_replications.return_value =
self.mock_describe_replication_response("started")
+
+ op = DmsStopReplicationOperator(
+ task_id="stop_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ wait_for_completion=False,
+ deferrable=False,
+ )
+ op.execute({})
+ assert mock_stop.call_count == 1
+
+ @mock.patch.object(DmsHook, "conn")
+ @mock.patch.object(DmsHook, "describe_replications")
+ @mock.patch.object(DmsHook, "get_waiter")
+ def test_wait_for_completion(self, mock_get_waiter,
mock_describe_replications, mock_conn):
+ mock_describe_replications.return_value =
self.mock_describe_replication_response("started")
+ op = DmsStopReplicationOperator(
+ task_id="stop_replication",
+ replication_config_arn="XXXXXXXXXXXXXXX",
+ wait_for_completion=True,
+ deferrable=False,
+ )
+
+ op.execute({})
+ mock_get_waiter.assert_called_with("replication_stopped")
+ mock_get_waiter.assert_called_once()
diff --git a/providers/tests/amazon/aws/triggers/test_dms.py
b/providers/tests/amazon/aws/triggers/test_dms.py
new file mode 100644
index 00000000000..da86e6a8e85
--- /dev/null
+++ b/providers/tests/amazon/aws/triggers/test_dms.py
@@ -0,0 +1,188 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from unittest import mock
+from unittest.mock import AsyncMock
+
+import pytest
+
+from airflow.providers.amazon.aws.hooks.dms import DmsHook
+from airflow.providers.amazon.aws.triggers.dms import (
+ DmsReplicationCompleteTrigger,
+ DmsReplicationConfigDeletedTrigger,
+ DmsReplicationDeprovisionedTrigger,
+ DmsReplicationStoppedTrigger,
+ DmsReplicationTerminalStatusTrigger,
+)
+from airflow.triggers.base import TriggerEvent
+
+from providers.tests.amazon.aws.utils.test_waiter import
assert_expected_waiter_type
+
+BASE_TRIGGER_CLASSPATH = "airflow.providers.amazon.aws.triggers.dms."
+
+
+class TestBaseDmsTrigger:
+ EXPECTED_WAITER_NAME: str | None = None
+
+ def test_setup(self):
+ if self.__class__.__name__ != "TestBaseDmsTrigger":
+ assert isinstance(self.EXPECTED_WAITER_NAME, str)
+
+
+class TestDmsReplicationCompleteTrigger(TestBaseDmsTrigger):
+ EXPECTED_WAITER_NAME = "replication_complete"
+ REPLICATION_CONFIG_ARN = "arn:aws:dms:region:account:config"
+
+ def test_serialization(self):
+ trigger =
DmsReplicationCompleteTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+
+ classpath, kwargs = trigger.serialize()
+ assert classpath == BASE_TRIGGER_CLASSPATH +
"DmsReplicationCompleteTrigger"
+
+ assert kwargs.get("replication_config_arn") ==
self.REPLICATION_CONFIG_ARN
+
+ @pytest.mark.asyncio
+ @mock.patch.object(DmsHook, "get_waiter")
+ @mock.patch.object(DmsHook, "get_conn")
+ async def test_complete(self, mock_async_conn, mock_get_waiter):
+ mock_async_conn.__aenter__.return_value = mock.MagicMock()
+ mock_get_waiter().wait = AsyncMock()
+ trigger =
DmsReplicationCompleteTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+ generator = trigger.run()
+ response = await generator.asend(None)
+ assert response == TriggerEvent(
+ {"status": "success", "replication_config_arn":
self.REPLICATION_CONFIG_ARN}
+ )
+ mock_get_waiter().wait.assert_called_once()
+
+
+class TestDmsReplicationTerminalStatusTrigger(TestBaseDmsTrigger):
+ EXPECTED_WAITER_NAME = "replication_terminal_status"
+ REPLICATION_CONFIG_ARN = "arn:aws:dms:region:account:config"
+
+ def test_serialization(self):
+ trigger =
DmsReplicationTerminalStatusTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+
+ classpath, kwargs = trigger.serialize()
+ assert classpath == BASE_TRIGGER_CLASSPATH +
"DmsReplicationTerminalStatusTrigger"
+
+ assert kwargs.get("replication_config_arn") ==
self.REPLICATION_CONFIG_ARN
+
+ @pytest.mark.asyncio
+ @mock.patch.object(DmsHook, "get_waiter")
+ @mock.patch.object(DmsHook, "get_conn")
+ async def test_complete(self, mock_async_conn, mock_get_waiter):
+ mock_async_conn.__aenter__.return_value = mock.MagicMock()
+ mock_get_waiter().wait = AsyncMock()
+ trigger =
DmsReplicationTerminalStatusTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+ generator = trigger.run()
+ response = await generator.asend(None)
+ assert response == TriggerEvent(
+ {"status": "success", "replication_config_arn":
self.REPLICATION_CONFIG_ARN}
+ )
+ assert_expected_waiter_type(mock_get_waiter, self.EXPECTED_WAITER_NAME)
+
+ mock_get_waiter().wait.assert_called_once()
+
+
+class TestDmsReplicationConfigDeletedTrigger(TestBaseDmsTrigger):
+ EXPECTED_WAITER_NAME = "replication_config_deleted"
+ REPLICATION_CONFIG_ARN = "arn:aws:dms:region:account:config"
+
+ def test_serialization(self):
+ trigger =
DmsReplicationConfigDeletedTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+
+ classpath, kwargs = trigger.serialize()
+ assert classpath == BASE_TRIGGER_CLASSPATH +
"DmsReplicationConfigDeletedTrigger"
+
+ assert kwargs.get("replication_config_arn") ==
self.REPLICATION_CONFIG_ARN
+
+ @pytest.mark.asyncio
+ @mock.patch.object(DmsHook, "get_waiter")
+ @mock.patch.object(DmsHook, "get_conn")
+ async def test_complete(self, mock_async_conn, mock_get_waiter):
+ mock_async_conn.__aenter__.return_value = mock.MagicMock()
+ mock_get_waiter().wait = AsyncMock()
+ trigger =
DmsReplicationConfigDeletedTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+ generator = trigger.run()
+ response = await generator.asend(None)
+ assert response == TriggerEvent(
+ {"status": "success", "replication_config_arn":
self.REPLICATION_CONFIG_ARN}
+ )
+ assert_expected_waiter_type(mock_get_waiter, self.EXPECTED_WAITER_NAME)
+
+ mock_get_waiter().wait.assert_called_once()
+
+
+class TestDmsReplicationStoppedTrigger(TestBaseDmsTrigger):
+ EXPECTED_WAITER_NAME = "replication_stopped"
+ REPLICATION_CONFIG_ARN = "arn:aws:dms:region:account:config"
+
+ def test_serialization(self):
+ trigger =
DmsReplicationStoppedTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+
+ classpath, kwargs = trigger.serialize()
+ assert classpath == BASE_TRIGGER_CLASSPATH +
"DmsReplicationStoppedTrigger"
+
+ """ assert kwargs.get("Filters") == [
+ {"Name": "replication-config-arn", "Values":
["arn:aws:dms:region:account:config"]}
+ ] """
+ assert kwargs.get("replication_config_arn") ==
self.REPLICATION_CONFIG_ARN
+
+ @pytest.mark.asyncio
+ @mock.patch.object(DmsHook, "get_waiter")
+ @mock.patch.object(DmsHook, "get_conn")
+ async def test_complete(self, mock_async_conn, mock_get_waiter):
+ mock_async_conn.__aenter__.return_value = mock.MagicMock()
+ mock_get_waiter().wait = AsyncMock()
+ trigger =
DmsReplicationStoppedTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+ generator = trigger.run()
+ response = await generator.asend(None)
+ assert response == TriggerEvent(
+ {"status": "success", "replication_config_arn":
self.REPLICATION_CONFIG_ARN}
+ )
+ assert_expected_waiter_type(mock_get_waiter, self.EXPECTED_WAITER_NAME)
+ mock_get_waiter().wait.assert_called_once()
+
+
+class TestDmsReplicationDeprovisionedTrigger(TestBaseDmsTrigger):
+ EXPECTED_WAITER_NAME = "replication_deprovisioned"
+ REPLICATION_CONFIG_ARN = "arn:aws:dms:region:account:config"
+
+ def test_serialization(self):
+ trigger =
DmsReplicationDeprovisionedTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+
+ classpath, kwargs = trigger.serialize()
+ assert classpath == BASE_TRIGGER_CLASSPATH +
"DmsReplicationDeprovisionedTrigger"
+
+ assert kwargs.get("replication_config_arn") ==
self.REPLICATION_CONFIG_ARN
+
+ @pytest.mark.asyncio
+ @mock.patch.object(DmsHook, "get_waiter")
+ @mock.patch.object(DmsHook, "get_conn")
+ async def test_complete(self, mock_async_conn, mock_get_waiter):
+ mock_async_conn.__aenter__.return_value = mock.MagicMock()
+ mock_get_waiter().wait = AsyncMock()
+ trigger =
DmsReplicationDeprovisionedTrigger(replication_config_arn=self.REPLICATION_CONFIG_ARN)
+ generator = trigger.run()
+ response = await generator.asend(None)
+ assert response == TriggerEvent(
+ {"status": "success", "replication_config_arn":
self.REPLICATION_CONFIG_ARN}
+ )
+ assert_expected_waiter_type(mock_get_waiter, self.EXPECTED_WAITER_NAME)
+ mock_get_waiter().wait.assert_called_once()
diff --git a/providers/tests/amazon/aws/waiters/test_dms.py
b/providers/tests/amazon/aws/waiters/test_dms.py
new file mode 100644
index 00000000000..86d36ee9c3e
--- /dev/null
+++ b/providers/tests/amazon/aws/waiters/test_dms.py
@@ -0,0 +1,139 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from unittest import mock
+
+import boto3
+import pytest
+
+from airflow.providers.amazon.aws.hooks.dms import DmsHook
+
+
+class TestCustomDmsWaiters:
+ """Test waiters from ``amazon/aws/waiters/dms.json```."""
+
+ @pytest.fixture(autouse=True)
+ def setup_test_cases(self, monkeypatch):
+ self.client = boto3.client("dms", region_name="us-east-1")
+ monkeypatch.setattr(DmsHook, "conn", self.client)
+
+ def test_service_waiters(self):
+ hook_waiters = DmsHook(aws_conn_id=None).list_waiters()
+ assert "replication_terminal_status" in hook_waiters
+ assert "replication_config_deleted" in hook_waiters
+ assert "replication_stopped" in hook_waiters
+ assert "replication_complete" in hook_waiters
+
+ @pytest.fixture
+ def mock_describe_replication(self):
+ with mock.patch.object(self.client, "describe_replications") as m:
+ yield m
+
+ @pytest.fixture
+ def mock_describe_replication_configs(self):
+ with mock.patch.object(self.client, "describe_replication_configs") as
m:
+ yield m
+
+ def test_wait_for_replication_terminal_status(self,
mock_describe_replication):
+ mock_describe_replication.return_value = {
+ "Replications": [
+ {
+ "ReplicationConfigArn":
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+ "Status": "failed",
+ }
+ ]
+ }
+
+ hook = DmsHook(aws_conn_id=None)
+ waiter = hook.get_waiter("replication_terminal_status")
+ waiter.wait(
+ Filters=[
+ {
+ "Name": "replication-instance-arn",
+ "Values": ["XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"],
+ }
+ ]
+ )
+
+ mock_describe_replication.assert_called_once_with(
+ Filters=[
+ {
+ "Name": "replication-instance-arn",
+ "Values": ["XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"],
+ }
+ ]
+ )
+
+ def test_wait_for_replication_config_delete(self,
mock_describe_replication_configs):
+ mock_describe_replication_configs.side_effect = [
+ {"Replications": [{"ReplicationArn": "MyArn"}]},
+ {"Error": {"Code": "ResourceNotFoundFault"}},
+ ]
+
+ hook = DmsHook(aws_conn_id=None)
+ waiter = hook.get_waiter("replication_config_deleted")
+ waiter.wait(
+ Filters=[
+ {
+ "Name": "replication-config-arn",
+ "Values": ["XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"],
+ }
+ ],
+ WaiterConfig={"Delay": 0.01, "MaxAttempts": 3},
+ )
+
+ mock_describe_replication_configs.assert_called_with(
+ Filters=[
+ {
+ "Name": "replication-config-arn",
+ "Values": ["XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"],
+ }
+ ]
+ )
+
+ def test_wait_for_replication_stopped(self, mock_describe_replication):
+ mock_describe_replication.return_value = {
+ "Replications": [
+ {
+ "ReplicationConfigArn":
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+ "Status": "stopped",
+ }
+ ]
+ }
+
+ hook = DmsHook(aws_conn_id=None)
+ waiter = hook.get_waiter("replication_stopped")
+ waiter.wait(
+ Filters=[
+ {
+ "Name": "replication_config_arn",
+ "Values": ["XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"],
+ }
+ ],
+ WaiterConfig={"Delay": 0.01, "MaxAttempts": 3},
+ )
+
+ mock_describe_replication.assert_called_once_with(
+ Filters=[
+ {
+ "Name": "replication_config_arn",
+ "Values": ["XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"],
+ }
+ ]
+ )
diff --git a/providers/tests/system/amazon/aws/example_dms_serverless.py
b/providers/tests/system/amazon/aws/example_dms_serverless.py
new file mode 100644
index 00000000000..4fca4f688a5
--- /dev/null
+++ b/providers/tests/system/amazon/aws/example_dms_serverless.py
@@ -0,0 +1,481 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Note: DMS requires you to configure specific IAM roles/permissions. For more
information, see
+https://docs.aws.amazon.com/dms/latest/userguide/security-iam.html#CHAP_Security.APIRole
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+
+import boto3
+from sqlalchemy import Column, MetaData, String, Table, create_engine
+
+from airflow.decorators import task
+from airflow.models.baseoperator import chain
+from airflow.models.dag import DAG
+from airflow.providers.amazon.aws.operators.dms import (
+ DmsCreateReplicationConfigOperator,
+ DmsDeleteReplicationConfigOperator,
+ DmsDescribeReplicationConfigsOperator,
+ DmsDescribeReplicationsOperator,
+ DmsStartReplicationOperator,
+ DmsStopReplicationOperator,
+)
+from airflow.providers.amazon.aws.operators.rds import (
+ RdsCreateDbInstanceOperator,
+ RdsDeleteDbInstanceOperator,
+)
+from airflow.providers.amazon.aws.operators.s3 import S3CreateBucketOperator,
S3DeleteBucketOperator
+from airflow.utils.trigger_rule import TriggerRule
+
+from providers.tests.system.amazon.aws.utils import ENV_ID_KEY,
SystemTestContextBuilder
+from providers.tests.system.amazon.aws.utils.ec2 import get_default_vpc_id
+
+"""
+This example demonstrates how to use the DMS operators to create a serverless
replication task to replicate data
+from a PostgreSQL database to Amazon S3.
+
+The IAM role used for the replication must have the permissions defined in the
[Amazon S3
target](https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Target.S3.html#CHAP_Target.S3.Prerequisites)
+documentation.
+"""
+
+DAG_ID = "example_dms_serverless"
+ROLE_ARN_KEY = "ROLE_ARN"
+
+sys_test_context_task =
SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build()
+
+# Config values for setting up the "Source" database.
+CA_CERT_ID = "rds-ca-rsa2048-g1"
+RDS_ENGINE = "postgres"
+RDS_PROTOCOL = "postgresql"
+RDS_USERNAME = "username"
+# NEVER store your production password in plaintext in a DAG like this.
+# Use Airflow Secrets or a secret manager for this in production.
+RDS_PASSWORD = "rds_password"
+TABLE_HEADERS = ["apache_project", "release_year"]
+SAMPLE_DATA = [
+ ("Airflow", "2015"),
+ ("OpenOffice", "2012"),
+ ("Subversion", "2000"),
+ ("NiFi", "2006"),
+]
+SG_IP_PERMISSION = {
+ "FromPort": 5432,
+ "IpProtocol": "All",
+ "IpRanges": [{"CidrIp": "0.0.0.0/0"}],
+}
+
+
+def _get_rds_instance_endpoint(instance_name: str):
+ print("Retrieving RDS instance endpoint.")
+ rds_client = boto3.client("rds")
+
+ response =
rds_client.describe_db_instances(DBInstanceIdentifier=instance_name)
+ rds_instance_endpoint = response["DBInstances"][0]["Endpoint"]
+ return rds_instance_endpoint
+
+
+@task
+def create_security_group(security_group_name: str, vpc_id: str):
+ client = boto3.client("ec2")
+ security_group = client.create_security_group(
+ GroupName=security_group_name,
+ Description="Created for DMS system test",
+ VpcId=vpc_id,
+ )
+ client.get_waiter("security_group_exists").wait(
+ GroupIds=[security_group["GroupId"]],
+ )
+ client.authorize_security_group_ingress(
+ GroupId=security_group["GroupId"],
+ IpPermissions=[SG_IP_PERMISSION],
+ )
+
+ return security_group["GroupId"]
+
+
+@task
+def create_sample_table(instance_name: str, db_name: str, table_name: str):
+ print("Creating sample table.")
+
+ rds_endpoint = _get_rds_instance_endpoint(instance_name)
+ hostname = rds_endpoint["Address"]
+ port = rds_endpoint["Port"]
+ rds_url =
f"{RDS_PROTOCOL}://{RDS_USERNAME}:{RDS_PASSWORD}@{hostname}:{port}/{db_name}"
+ engine = create_engine(rds_url)
+
+ table = Table(
+ table_name,
+ MetaData(engine),
+ Column(TABLE_HEADERS[0], String, primary_key=True),
+ Column(TABLE_HEADERS[1], String),
+ )
+
+ with engine.connect() as connection:
+ # Create the Table.
+ table.create()
+ load_data = table.insert().values(SAMPLE_DATA)
+ connection.execute(load_data)
+
+ # Read the data back to verify everything is working.
+ connection.execute(table.select())
+
+
+@task(trigger_rule=TriggerRule.ALL_SUCCESS)
+def create_vpc_endpoints(vpc_id: str):
+ print("Creating VPC endpoints in vpc: %s", vpc_id)
+ client = boto3.client("ec2")
+ session = boto3.session.Session()
+ region = session.region_name
+ route_tbls = client.describe_route_tables(Filters=[{"Name": "vpc-id",
"Values": [vpc_id]}])
+ endpoints = client.create_vpc_endpoint(
+ VpcId=vpc_id,
+ ServiceName=f"com.amazonaws.{region}.s3",
+ VpcEndpointType="Gateway",
+ RouteTableIds=[tbl["RouteTableId"] for tbl in
route_tbls["RouteTables"]],
+ )
+
+ return endpoints.get("VpcEndpoint", {}).get("VpcEndpointId")
+
+
+@task(trigger_rule=TriggerRule.ALL_DONE)
+def delete_vpc_endpoints(endpoint_ids: list[str]):
+ if len(endpoint_ids) == 0:
+ print("No VPC endpoints to delete.")
+ return
+
+ print("Deleting VPC endpoints.")
+ client = boto3.client("ec2")
+
+ client.delete_vpc_endpoints(VpcEndpointIds=endpoint_ids, DryRun=False)
+
+ print("Deleted endpoints: %s", endpoint_ids)
+
+
+@task(multiple_outputs=True)
+def create_dms_assets(
+ db_name: str,
+ instance_name: str,
+ bucket_name: str,
+ role_arn,
+ source_endpoint_identifier: str,
+ target_endpoint_identifier: str,
+ table_definition: dict,
+):
+ print("Creating DMS assets.")
+ dms_client = boto3.client("dms")
+ rds_instance_endpoint = _get_rds_instance_endpoint(instance_name)
+
+ print("Creating DMS source endpoint.")
+ source_endpoint_arn = dms_client.create_endpoint(
+ EndpointIdentifier=source_endpoint_identifier,
+ EndpointType="source",
+ EngineName=RDS_ENGINE,
+ Username=RDS_USERNAME,
+ Password=RDS_PASSWORD,
+ ServerName=rds_instance_endpoint["Address"],
+ Port=rds_instance_endpoint["Port"],
+ DatabaseName=db_name,
+ SslMode="require",
+ )["Endpoint"]["EndpointArn"]
+
+ print("Creating DMS target endpoint.")
+ target_endpoint_arn = dms_client.create_endpoint(
+ EndpointIdentifier=target_endpoint_identifier,
+ EndpointType="target",
+ EngineName="s3",
+ S3Settings={
+ "BucketName": bucket_name,
+ "BucketFolder": "folder",
+ "ServiceAccessRoleArn": role_arn,
+ "ExternalTableDefinition": json.dumps(table_definition),
+ },
+ )["Endpoint"]["EndpointArn"]
+
+ return {
+ "source_endpoint_arn": source_endpoint_arn,
+ "target_endpoint_arn": target_endpoint_arn,
+ }
+
+
+@task(trigger_rule=TriggerRule.ALL_DONE)
+def delete_dms_assets(
+ source_endpoint_arn: str,
+ target_endpoint_arn: str,
+ source_endpoint_identifier: str,
+ target_endpoint_identifier: str,
+):
+ dms_client = boto3.client("dms")
+
+ print("Deleting DMS assets.")
+
+ print(source_endpoint_arn)
+ print(target_endpoint_arn)
+
+ try:
+ dms_client.delete_endpoint(EndpointArn=source_endpoint_arn)
+ dms_client.delete_endpoint(EndpointArn=target_endpoint_arn)
+ except Exception as ex:
+ print("Exception while cleaning up endpoints:%s", ex)
+
+ print("Awaiting DMS assets tear-down.")
+
+ dms_client.get_waiter("endpoint_deleted").wait(
+ Filters=[
+ {
+ "Name": "endpoint-id",
+ "Values": [source_endpoint_identifier,
target_endpoint_identifier],
+ }
+ ]
+ )
+
+
+@task(trigger_rule=TriggerRule.ALL_DONE)
+def delete_security_group(security_group_id: str, security_group_name: str):
+ boto3.client("ec2").delete_security_group(GroupId=security_group_id,
GroupName=security_group_name)
+
+
+# setup
+# source: aurora serverless
+# dest: S3
+# S3
+
+with DAG(
+ dag_id=DAG_ID,
+ schedule="@once",
+ start_date=datetime(2021, 1, 1),
+ tags=["example"],
+ catchup=False,
+) as dag:
+ test_context = sys_test_context_task()
+ env_id = test_context[ENV_ID_KEY]
+ role_arn = test_context[ROLE_ARN_KEY]
+
+ bucket_name = f"{env_id}-dms-bucket"
+ rds_instance_name = f"{env_id}-instance"
+ rds_db_name = f"{env_id}_source_database" # dashes are not allowed in db
name
+ rds_table_name = f"{env_id}-table"
+ dms_replication_instance_name = f"{env_id}-replication-instance"
+ dms_replication_task_id = f"{env_id}-replication-task"
+ source_endpoint_identifier = f"{env_id}-source-endpoint"
+ target_endpoint_identifier = f"{env_id}-target-endpoint"
+ security_group_name = f"{env_id}-dms-security-group"
+ replication_id = f"{env_id}-replication-id"
+
+ create_s3_bucket = S3CreateBucketOperator(task_id="create_s3_bucket",
bucket_name=bucket_name)
+
+ get_vpc_id = get_default_vpc_id()
+
+ create_sg = create_security_group(security_group_name, get_vpc_id)
+
+ create_db_instance = RdsCreateDbInstanceOperator(
+ task_id="create_db_instance",
+ db_instance_identifier=rds_instance_name,
+ db_instance_class="db.t3.micro",
+ engine=RDS_ENGINE,
+ rds_kwargs={
+ "DBName": rds_db_name,
+ "AllocatedStorage": 20,
+ "MasterUsername": RDS_USERNAME,
+ "MasterUserPassword": RDS_PASSWORD,
+ "PubliclyAccessible": True,
+ "VpcSecurityGroupIds": [
+ create_sg,
+ ],
+ },
+ )
+
+ # Sample data.
+ table_definition = {
+ "TableCount": "1",
+ "Tables": [
+ {
+ "TableName": rds_table_name,
+ "TableColumns": [
+ {
+ "ColumnName": TABLE_HEADERS[0],
+ "ColumnType": "STRING",
+ "ColumnNullable": "false",
+ "ColumnIsPk": "true",
+ },
+ {"ColumnName": TABLE_HEADERS[1], "ColumnType": "STRING",
"ColumnLength": "4"},
+ ],
+ "TableColumnsTotal": "2",
+ }
+ ],
+ }
+ table_mappings = {
+ "rules": [
+ {
+ "rule-type": "selection",
+ "rule-id": "1",
+ "rule-name": "1",
+ "object-locator": {
+ "schema-name": "public",
+ "table-name": rds_table_name,
+ },
+ "rule-action": "include",
+ }
+ ]
+ }
+
+ create_assets = create_dms_assets(
+ db_name=rds_db_name,
+ instance_name=rds_instance_name,
+ bucket_name=bucket_name,
+ role_arn=role_arn,
+ source_endpoint_identifier=source_endpoint_identifier,
+ target_endpoint_identifier=target_endpoint_identifier,
+ table_definition=table_definition,
+ )
+
+ # [START howto_operator_dms_create_replication_config]
+ create_replication_config = DmsCreateReplicationConfigOperator(
+ task_id="create_replication_config",
+ replication_config_id=replication_id,
+ source_endpoint_arn=create_assets["source_endpoint_arn"],
+ target_endpoint_arn=create_assets["target_endpoint_arn"],
+ compute_config={
+ "MaxCapacityUnits": 4,
+ "MinCapacityUnits": 1,
+ "MultiAZ": False,
+ "ReplicationSubnetGroupId": "default",
+ },
+ replication_type="full-load",
+ table_mappings=json.dumps(table_mappings),
+ trigger_rule=TriggerRule.ALL_SUCCESS,
+ )
+ # [END howto_operator_dms_create_replication_config]
+
+ # [START howto_operator_dms_describe_replication_config]
+ describe_replication_configs = DmsDescribeReplicationConfigsOperator(
+ task_id="describe_replication_configs",
+ trigger_rule=TriggerRule.ALL_SUCCESS,
+ )
+ # [END howto_operator_dms_describe_replication_config]
+
+ # [START howto_operator_dms_serverless_describe_replication]
+ describe_replications = DmsDescribeReplicationsOperator(
+ task_id="describe_replications",
+ trigger_rule=TriggerRule.ALL_SUCCESS,
+ )
+ # [END howto_operator_dms_serverless_describe_replication]
+
+ # [START howto_operator_dms_serverless_start_replication]
+ replicate = DmsStartReplicationOperator(
+ task_id="replicate",
+ replication_config_arn="{{
task_instance.xcom_pull(task_ids='create_replication_config',
key='return_value') }}",
+ replication_start_type="start-replication",
+ wait_for_completion=True,
+ waiter_delay=60,
+ waiter_max_attempts=200,
+ trigger_rule=TriggerRule.ALL_SUCCESS,
+ deferrable=False,
+ )
+ # [END howto_operator_dms_serverless_start_replication]
+
+ # [START howto_operator_dms_serverless_stop_replication]
+ stop_relication = DmsStopReplicationOperator(
+ task_id="stop_replication",
+ replication_config_arn="{{
task_instance.xcom_pull(task_ids='create_replication_config',
key='return_value') }}",
+ wait_for_completion=True,
+ waiter_delay=120,
+ waiter_max_attempts=200,
+ trigger_rule=TriggerRule.ALL_SUCCESS,
+ deferrable=False,
+ )
+ # [END howto_operator_dms_serverless_stop_replication]
+
+ # [START howto_operator_dms_serverless_delete_replication_config]
+ delete_replication_config = DmsDeleteReplicationConfigOperator(
+ task_id="delete_replication_config",
+ wait_for_completion=True,
+ waiter_delay=60,
+ waiter_max_attempts=200,
+ deferrable=False,
+ replication_config_arn="{{
task_instance.xcom_pull(task_ids='create_replication_config',
key='return_value') }}",
+ trigger_rule=TriggerRule.ALL_DONE,
+ )
+ # [END howto_operator_dms_serverless_delete_replication_config]
+
+ delete_assets = delete_dms_assets(
+ source_endpoint_arn=create_assets["source_endpoint_arn"],
+ target_endpoint_arn=create_assets["target_endpoint_arn"],
+ source_endpoint_identifier=source_endpoint_identifier,
+ target_endpoint_identifier=target_endpoint_identifier,
+ )
+
+ delete_db_instance = RdsDeleteDbInstanceOperator(
+ task_id="delete_db_instance",
+ db_instance_identifier=rds_instance_name,
+ rds_kwargs={
+ "SkipFinalSnapshot": True,
+ },
+ trigger_rule=TriggerRule.ALL_DONE,
+ )
+
+ delete_s3_bucket = S3DeleteBucketOperator(
+ task_id="delete_s3_bucket",
+ bucket_name=bucket_name,
+ force_delete=True,
+ trigger_rule=TriggerRule.ALL_DONE,
+ )
+
+ chain(
+ # TEST SETUP
+ create_s3_bucket,
+ get_vpc_id,
+ create_sg,
+ create_db_instance,
+ create_sample_table(rds_instance_name, rds_db_name, rds_table_name),
+ create_vpc_endpoints(
+ vpc_id="{{
task_instance.xcom_pull(task_ids='get_default_vpc_id',key='return_value')}}"
+ ),
+ create_assets,
+ # TEST BODY
+ create_replication_config,
+ describe_replication_configs,
+ replicate,
+ stop_relication,
+ describe_replications,
+ delete_replication_config,
+ # TEST TEARDOWN
+ delete_vpc_endpoints(
+ endpoint_ids=[
+ "{{ task_instance.xcom_pull(task_ids='create_vpc_endpoints',
key='return_value') }}"
+ ]
+ ),
+ delete_assets,
+ delete_db_instance,
+ delete_security_group(create_sg, security_group_name),
+ delete_s3_bucket,
+ )
+
+ from tests_common.test_utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+from tests_common.test_utils.system_tests import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see:
tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)