Lee-W commented on code in PR #42080:
URL: https://github.com/apache/airflow/pull/42080#discussion_r1770727553
##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,41 @@ def execute(self, context: Context) -> None:
subscription=self.subscription,
max_messages=self.max_messages,
ack_messages=self.ack_messages,
- messages_callback=self.messages_callback,
poke_interval=self.poke_interval,
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
),
method_name="execute_complete",
)
- def execute_complete(self, context: dict[str, Any], event: dict[str, str |
list[str]]) -> str | list[str]:
+ def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> Any:
Review Comment:
```suggestion
def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> list[Any]:
```
##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,41 @@ def execute(self, context: Context) -> None:
subscription=self.subscription,
max_messages=self.max_messages,
ack_messages=self.ack_messages,
- messages_callback=self.messages_callback,
poke_interval=self.poke_interval,
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
),
method_name="execute_complete",
)
- def execute_complete(self, context: dict[str, Any], event: dict[str, str |
list[str]]) -> str | list[str]:
+ def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> Any:
"""Return immediately and relies on trigger to throw a success event.
Callback for the trigger."""
Review Comment:
We probably would like to update the docstring as we're doing something
before returning
##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,41 @@ def execute(self, context: Context) -> None:
subscription=self.subscription,
max_messages=self.max_messages,
ack_messages=self.ack_messages,
- messages_callback=self.messages_callback,
poke_interval=self.poke_interval,
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
),
method_name="execute_complete",
)
- def execute_complete(self, context: dict[str, Any], event: dict[str, str |
list[str]]) -> str | list[str]:
+ def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> Any:
"""Return immediately and relies on trigger to throw a success event.
Callback for the trigger."""
if event["status"] == "success":
self.log.info("Sensor pulls messages: %s", event["message"])
+ if self.messages_callback:
+ received_message_fmt =
self._convert_to_received_message(event["message"])
+ message_callback_response =
self.messages_callback(received_message_fmt, context)
+
+ return message_callback_response
+
return event["message"]
self.log.info("Sensor failed: %s", event["message"])
raise AirflowException(event["message"])
+ def _convert_to_received_message(self, messages: Any):
+ try:
+ received_messages = []
+ for msg in messages:
+ received_message = pubsub_v1.types.ReceivedMessage(msg)
+
+ received_messages.append(received_message)
Review Comment:
```suggestion
received_messages = [pubsub_v1.types.ReceivedMessage(msg) for
msg in messages]
```
##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,41 @@ def execute(self, context: Context) -> None:
subscription=self.subscription,
max_messages=self.max_messages,
ack_messages=self.ack_messages,
- messages_callback=self.messages_callback,
poke_interval=self.poke_interval,
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
),
method_name="execute_complete",
)
- def execute_complete(self, context: dict[str, Any], event: dict[str, str |
list[str]]) -> str | list[str]:
+ def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> Any:
"""Return immediately and relies on trigger to throw a success event.
Callback for the trigger."""
if event["status"] == "success":
self.log.info("Sensor pulls messages: %s", event["message"])
+ if self.messages_callback:
+ received_message_fmt =
self._convert_to_received_message(event["message"])
Review Comment:
May I know why can't we just reuse the logic
[here](https://github.com/apache/airflow/blob/111c6327bb5c2b4dd4421c8557b8b7de670a01e3/airflow/providers/google/cloud/sensors/pubsub.py#L148C1-L148C83)?
##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,41 @@ def execute(self, context: Context) -> None:
subscription=self.subscription,
max_messages=self.max_messages,
ack_messages=self.ack_messages,
- messages_callback=self.messages_callback,
poke_interval=self.poke_interval,
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
),
method_name="execute_complete",
)
- def execute_complete(self, context: dict[str, Any], event: dict[str, str |
list[str]]) -> str | list[str]:
+ def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> Any:
"""Return immediately and relies on trigger to throw a success event.
Callback for the trigger."""
if event["status"] == "success":
self.log.info("Sensor pulls messages: %s", event["message"])
+ if self.messages_callback:
+ received_message_fmt =
self._convert_to_received_message(event["message"])
+ message_callback_response =
self.messages_callback(received_message_fmt, context)
+
+ return message_callback_response
+
return event["message"]
self.log.info("Sensor failed: %s", event["message"])
raise AirflowException(event["message"])
+ def _convert_to_received_message(self, messages: Any):
Review Comment:
It looks like returning a list of any, but doesn't look like the same
behavior as `self._return_value` 🤔 Could you please elaborate a bit?
##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,41 @@ def execute(self, context: Context) -> None:
subscription=self.subscription,
max_messages=self.max_messages,
ack_messages=self.ack_messages,
- messages_callback=self.messages_callback,
poke_interval=self.poke_interval,
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
),
method_name="execute_complete",
)
- def execute_complete(self, context: dict[str, Any], event: dict[str, str |
list[str]]) -> str | list[str]:
+ def execute_complete(self, context: Context, event: dict[str, str |
list[str]]) -> Any:
"""Return immediately and relies on trigger to throw a success event.
Callback for the trigger."""
if event["status"] == "success":
self.log.info("Sensor pulls messages: %s", event["message"])
+ if self.messages_callback:
+ received_message_fmt =
self._convert_to_received_message(event["message"])
+ message_callback_response =
self.messages_callback(received_message_fmt, context)
+
+ return message_callback_response
+
return event["message"]
self.log.info("Sensor failed: %s", event["message"])
raise AirflowException(event["message"])
+ def _convert_to_received_message(self, messages: Any):
Review Comment:
```suggestion
def _convert_received_messages(self, messages: Any) -> list[Any]:
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]