vincbeck commented on code in PR #51719:
URL: https://github.com/apache/airflow/pull/51719#discussion_r2237482111
##########
providers/amazon/src/airflow/providers/amazon/aws/sensors/mwaa.py:
##########
@@ -150,8 +150,148 @@ def execute(self, context: Context):
external_dag_run_id=self.external_dag_run_id,
success_states=self.success_states,
failure_states=self.failure_states,
- # somehow the type of poke_interval is derived as float ??
- waiter_delay=self.poke_interval, # type: ignore[arg-type]
+ waiter_delay=int(self.poke_interval),
+ waiter_max_attempts=self.max_retries,
+ aws_conn_id=self.aws_conn_id,
+ ),
+ method_name="execute_complete",
+ )
+ else:
+ super().execute(context=context)
+
+
+class MwaaTaskSensor(AwsBaseSensor[MwaaHook]):
+ """
+ Waits for a task in an MWAA Environment to complete.
+
+ If the task fails, an AirflowException is thrown.
+
+ .. seealso::
+ For more information on how to use this sensor, take a look at the
guide:
+ :ref:`howto/sensor:MwaaTaskSensor`
+
+ :param external_env_name: The external MWAA environment name that contains
the Task Instance you want to wait for
+ (templated)
+ :param external_dag_id: The DAG ID in the external MWAA environment that
contains the Task Instance you want to wait for
+ (templated)
+ :param external_dag_run_id: The DAG Run ID in the external MWAA
environment that you want to wait for (templated)
+ :param external_task_id: The Task ID in the external MWAA environment that
you want to wait for (templated)
+ :param success_states: Collection of task instance states that would make
this task marked as successful, default is
+ ``{airflow.utils.state.TaskInstanceState.SUCCESS}`` (templated)
+ :param failure_states: Collection of task instance states that would make
this task marked as failed and raise an
+ AirflowException, default is
``{airflow.utils.state.TaskInstanceState.FAILED}`` (templated)
+ :param deferrable: If True, the sensor will operate in deferrable mode.
This mode requires aiobotocore
+ module to be installed.
+ (default: False, but can be overridden in config file by setting
default_deferrable to True)
+ :param poke_interval: Polling period in seconds to check for the status of
the job. (default: 60)
+ :param max_retries: Number of times before returning the current state.
(default: 720)
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
+ If this is ``None`` or empty then the default boto3 behaviour is used.
If
+ running Airflow in a distributed manner and aws_conn_id is None or
+ empty, then default boto3 configuration would be used (and must be
+ maintained on each worker node).
+ :param region_name: AWS region_name. If not specified then the default
boto3 behaviour is used.
+ :param verify: Whether or not to verify SSL certificates. See:
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
+ :param botocore_config: Configuration dictionary (key-values) for botocore
client. See:
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
+ """
+
+ aws_hook_class = MwaaHook
+ template_fields: Sequence[str] = aws_template_fields(
+ "external_env_name",
+ "external_dag_id",
+ "external_dag_run_id",
+ "external_task_id",
+ "success_states",
+ "failure_states",
+ "deferrable",
+ "max_retries",
+ "poke_interval",
+ )
+
+ def __init__(
+ self,
+ *,
+ external_env_name: str,
+ external_dag_id: str,
+ external_dag_run_id: str | None = None,
+ external_task_id: str,
+ success_states: Collection[str] | None = None,
+ failure_states: Collection[str] | None = None,
+ deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ poke_interval: int = 60,
+ max_retries: int = 720,
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+
+ self.success_states = set(success_states) if success_states else
{TaskInstanceState.SUCCESS.value}
+ self.failure_states = set(failure_states) if failure_states else
{TaskInstanceState.FAILED.value}
+
+ if len(self.success_states & self.failure_states):
+ raise ValueError("success_states and failure_states must not have
any values in common")
+
+ self.external_env_name = external_env_name
+ self.external_dag_id = external_dag_id
+ self.external_dag_run_id = external_dag_run_id
+ self.external_task_id = external_task_id
+ self.deferrable = deferrable
+ self.poke_interval = poke_interval
+ self.max_retries = max_retries
+
+ def poke(self, context: Context) -> bool:
+ self.log.info(
+ "Poking for task %s of DAG run %s of DAG %s in MWAA environment
%s",
+ self.external_task_id,
+ self.external_dag_run_id,
+ self.external_dag_id,
+ self.external_env_name,
+ )
+
+ response = self.hook.invoke_rest_api(
+ env_name=self.external_env_name,
+
path=f"/dags/{self.external_dag_id}/dagRuns/{self.external_dag_run_id}/taskInstances/{self.external_task_id}",
+ method="GET",
+ )
+ # If RestApiStatusCode == 200, the RestApiResponse must have the
"state" key, otherwise something terrible has
+ # happened in the API and KeyError would be raised
+ # If RestApiStatusCode >= 300, a botocore exception would've already
been raised during the
+ # self.hook.invoke_rest_api call
+ # The scope of this sensor is going to only be raising
AirflowException due to failure of the task
+
+ state = response["RestApiResponse"]["state"]
+
+ if state in self.failure_states:
+ raise AirflowException(
+ f"The task {self.external_task_id} of DAG run
{self.external_dag_run_id} of DAG {self.external_dag_id} in MWAA environment
{self.external_env_name} "
+ f"failed with state: {state}"
+ )
+
+ return state in self.success_states
+
+ def execute_complete(self, context: Context, event: dict[str, Any] | None
= None) -> None:
+ validate_execute_complete_event(event)
Review Comment:
We do not do any action besides validating the event. Is it needed? Another
alternative to look at is, if validating the event is not necessary, we can end
the task in the triggerer without having to go back to the worker using
`end_from_trigger` attribute.
[Documentation](https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/deferring.html#exiting-deferred-task-from-triggers)
##########
providers/amazon/tests/system/amazon/aws/example_mwaa.py:
##########
@@ -118,25 +121,52 @@ def test_iam_fallback(role_to_assume_arn, mwaa_env_name):
)
# [END howto_operator_mwaa_trigger_dag_run]
+ # [START howto_sensor_mwaa_task]
+ wait_for_task = MwaaTaskSensor(
+ task_id="wait_for_task",
+ external_env_name=env_name,
+ external_dag_id=trigger_dag_id,
+ external_task_id=task_id,
+ poke_interval=5,
+ deferrable=True,
+ )
+ # [END howto_sensor_mwaa_task]
+
# [START howto_sensor_mwaa_dag_run]
wait_for_dag_run = MwaaDagRunSensor(
task_id="wait_for_dag_run",
external_env_name=env_name,
external_dag_id=trigger_dag_id,
external_dag_run_id="{{
task_instance.xcom_pull(task_ids='trigger_dag_run')['RestApiResponse']['dag_run_id']
}}",
poke_interval=5,
+ deferrable=True,
Review Comment:
Why hardcoding `deferrable` to `True`? We usually do not do that
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
