Lee-W commented on code in PR #42080:
URL: https://github.com/apache/airflow/pull/42080#discussion_r1770989322


##########
tests/providers/google/cloud/sensors/test_pubsub.py:
##########
@@ -197,3 +198,53 @@ def test_pubsub_pull_sensor_async_execute_complete(self):
         with mock.patch.object(operator.log, "info") as mock_log_info:
             operator.execute_complete(context={}, event={"status": "success", 
"message": test_message})
         mock_log_info.assert_called_with("Sensor pulls messages: %s", 
test_message)
+
+    @mock.patch("airflow.providers.google.cloud.sensors.pubsub.PubSubHook")
+    def 
test_pubsub_pull_sensor_async_execute_complete_use_message_callback(self, 
mock_hook):
+        test_message = [
+            {
+                "ack_id": 
"UAYWLF1GSFE3GQhoUQ5PXiM_NSAoRRIJB08CKF15MU0sQVhwaFENGXJ9YHxrUxsDV0ECel1RGQdoTm11H4GglfRLQ1RrWBIHB01Vel5TEwxoX11wBnm4vPO6v8vgfwk9OpX-8tltO6ywsP9GZiM9XhJLLD5-LzlFQV5AEkwkDERJUytDCypYEU4EISE-MD5FU0Q",
+                "message": {
+                    "data": "aGkgZnJvbSBjbG91ZCBjb25zb2xlIQ==",
+                    "message_id": "12165864188103151",
+                    "publish_time": "2024-08-28T11:49:50.962Z",
+                    "attributes": {},
+                    "ordering_key": "",
+                },
+                "delivery_attempt": 0,
+            }
+        ]
+        received_message_format = []
+        for msg in test_message:
+            
received_message_format.append(pubsub_v1.types.ReceivedMessage(msg))
+
+        messages_callback_return_value = "custom_message_from_callback"
+
+        def messages_callback(
+            pulled_messages: list[ReceivedMessage],
+            context: dict[str, Any],
+        ):
+            assert pulled_messages == received_message_format
+
+            assert isinstance(context, dict)
+            for key in context.keys():
+                assert isinstance(key, str)
+
+            return messages_callback_return_value
+
+        messages_callback = mock.Mock(side_effect=messages_callback)

Review Comment:
   What would happen if we remove this?



##########
tests/providers/google/cloud/sensors/test_pubsub.py:
##########
@@ -197,3 +198,53 @@ def test_pubsub_pull_sensor_async_execute_complete(self):
         with mock.patch.object(operator.log, "info") as mock_log_info:
             operator.execute_complete(context={}, event={"status": "success", 
"message": test_message})
         mock_log_info.assert_called_with("Sensor pulls messages: %s", 
test_message)
+
+    @mock.patch("airflow.providers.google.cloud.sensors.pubsub.PubSubHook")
+    def 
test_pubsub_pull_sensor_async_execute_complete_use_message_callback(self, 
mock_hook):
+        test_message = [
+            {
+                "ack_id": 
"UAYWLF1GSFE3GQhoUQ5PXiM_NSAoRRIJB08CKF15MU0sQVhwaFENGXJ9YHxrUxsDV0ECel1RGQdoTm11H4GglfRLQ1RrWBIHB01Vel5TEwxoX11wBnm4vPO6v8vgfwk9OpX-8tltO6ywsP9GZiM9XhJLLD5-LzlFQV5AEkwkDERJUytDCypYEU4EISE-MD5FU0Q",
+                "message": {
+                    "data": "aGkgZnJvbSBjbG91ZCBjb25zb2xlIQ==",
+                    "message_id": "12165864188103151",
+                    "publish_time": "2024-08-28T11:49:50.962Z",
+                    "attributes": {},
+                    "ordering_key": "",
+                },
+                "delivery_attempt": 0,
+            }
+        ]
+        received_message_format = []
+        for msg in test_message:
+            
received_message_format.append(pubsub_v1.types.ReceivedMessage(msg))

Review Comment:
   ```suggestion
           received_messages = [pubsub_v1.types.ReceivedMessage(msg) for msg in 
test_message]
   ```



##########
tests/providers/google/cloud/sensors/test_pubsub.py:
##########
@@ -197,3 +198,53 @@ def test_pubsub_pull_sensor_async_execute_complete(self):
         with mock.patch.object(operator.log, "info") as mock_log_info:
             operator.execute_complete(context={}, event={"status": "success", 
"message": test_message})
         mock_log_info.assert_called_with("Sensor pulls messages: %s", 
test_message)
+
+    @mock.patch("airflow.providers.google.cloud.sensors.pubsub.PubSubHook")
+    def 
test_pubsub_pull_sensor_async_execute_complete_use_message_callback(self, 
mock_hook):
+        test_message = [
+            {
+                "ack_id": 
"UAYWLF1GSFE3GQhoUQ5PXiM_NSAoRRIJB08CKF15MU0sQVhwaFENGXJ9YHxrUxsDV0ECel1RGQdoTm11H4GglfRLQ1RrWBIHB01Vel5TEwxoX11wBnm4vPO6v8vgfwk9OpX-8tltO6ywsP9GZiM9XhJLLD5-LzlFQV5AEkwkDERJUytDCypYEU4EISE-MD5FU0Q",

Review Comment:
   Is it a random `ack_id`? 



##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,36 @@ def execute(self, context: Context) -> None:
                     subscription=self.subscription,
                     max_messages=self.max_messages,
                     ack_messages=self.ack_messages,
-                    messages_callback=self.messages_callback,
                     poke_interval=self.poke_interval,
                     gcp_conn_id=self.gcp_conn_id,
                     impersonation_chain=self.impersonation_chain,
                 ),
                 method_name="execute_complete",
             )
 
-    def execute_complete(self, context: dict[str, Any], event: dict[str, str | 
list[str]]) -> str | list[str]:
-        """Return immediately and relies on trigger to throw a success event. 
Callback for the trigger."""
+    def execute_complete(self, context: Context, event: dict[str, str | 
list[str]]) -> Any:
+        """If messages_callback is provided, execute it; otherwise, return 
immediately with trigger event message."""
         if event["status"] == "success":
             self.log.info("Sensor pulls messages: %s", event["message"])
+            if self.messages_callback:
+                received_message_fmt = 
self._convert_to_received_message(event["message"])
+                message_callback_response = 
self.messages_callback(received_message_fmt, context)
+
+                return message_callback_response
+
             return event["message"]
         self.log.info("Sensor failed: %s", event["message"])
         raise AirflowException(event["message"])
 
+    def _convert_to_received_message(self, messages: Any) -> list[Any]:

Review Comment:
   ```suggestion
       def _convert_to_received_message(self, messages: Any) -> 
list[ReceivedMessage]:
   ```
   
   Sorry for the wrong suggestion previously. 🤦‍♂️ I was a bit confused 



##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,36 @@ def execute(self, context: Context) -> None:
                     subscription=self.subscription,
                     max_messages=self.max_messages,
                     ack_messages=self.ack_messages,
-                    messages_callback=self.messages_callback,
                     poke_interval=self.poke_interval,
                     gcp_conn_id=self.gcp_conn_id,
                     impersonation_chain=self.impersonation_chain,
                 ),
                 method_name="execute_complete",
             )
 
-    def execute_complete(self, context: dict[str, Any], event: dict[str, str | 
list[str]]) -> str | list[str]:
-        """Return immediately and relies on trigger to throw a success event. 
Callback for the trigger."""
+    def execute_complete(self, context: Context, event: dict[str, str | 
list[str]]) -> Any:
+        """If messages_callback is provided, execute it; otherwise, return 
immediately with trigger event message."""
         if event["status"] == "success":
             self.log.info("Sensor pulls messages: %s", event["message"])
+            if self.messages_callback:
+                received_message_fmt = 
self._convert_to_received_message(event["message"])
+                message_callback_response = 
self.messages_callback(received_message_fmt, context)
+
+                return message_callback_response

Review Comment:
   ```suggestion
                   received_messages = 
self._convert_to_received_messages(event["message"])
                   _return_value = self.messages_callback(received_messages, 
context)
   
                   return _return_value
   ```
   
   changing a few names to keep consistency as possible



##########
tests/providers/google/cloud/triggers/test_pubsub.py:
##########
@@ -34,13 +38,25 @@ def trigger():
         subscription="subscription",
         max_messages=MAX_MESSAGES,
         ack_messages=ACK_MESSAGES,
-        messages_callback=None,
         poke_interval=TEST_POLL_INTERVAL,
         gcp_conn_id=TEST_GCP_CONN_ID,
         impersonation_chain=None,
     )
 
 
+async def generate_messages(count):

Review Comment:
   ```suggestion
   async def generate_messages(count: int) -> list[ReceivedMessage]:
   ```



##########
airflow/providers/google/cloud/sensors/pubsub.py:
##########
@@ -170,22 +171,36 @@ def execute(self, context: Context) -> None:
                     subscription=self.subscription,
                     max_messages=self.max_messages,
                     ack_messages=self.ack_messages,
-                    messages_callback=self.messages_callback,
                     poke_interval=self.poke_interval,
                     gcp_conn_id=self.gcp_conn_id,
                     impersonation_chain=self.impersonation_chain,
                 ),
                 method_name="execute_complete",
             )
 
-    def execute_complete(self, context: dict[str, Any], event: dict[str, str | 
list[str]]) -> str | list[str]:
-        """Return immediately and relies on trigger to throw a success event. 
Callback for the trigger."""
+    def execute_complete(self, context: Context, event: dict[str, str | 
list[str]]) -> Any:
+        """If messages_callback is provided, execute it; otherwise, return 
immediately with trigger event message."""
         if event["status"] == "success":
             self.log.info("Sensor pulls messages: %s", event["message"])
+            if self.messages_callback:
+                received_message_fmt = 
self._convert_to_received_message(event["message"])
+                message_callback_response = 
self.messages_callback(received_message_fmt, context)
+
+                return message_callback_response
+
             return event["message"]
         self.log.info("Sensor failed: %s", event["message"])
         raise AirflowException(event["message"])
 
+    def _convert_to_received_message(self, messages: Any) -> list[Any]:
+        try:
+            received_messages = [pubsub_v1.types.ReceivedMessage(msg) for msg 
in messages]
+            return received_messages
+        except Exception as e:
+            raise AirflowException(

Review Comment:
   Instead of using `AirflowExecption`, it's better for us to inherit from 
`AirflowException` and create a customized exception



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to