This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new c434c6b989 Create `CloudComposerRunAirflowCLICommandOperator` operator 
(#38965)
c434c6b989 is described below

commit c434c6b98911d71f27910d7d30cf966462157612
Author: Maksim <[email protected]>
AuthorDate: Wed Apr 24 14:25:39 2024 +0200

    Create `CloudComposerRunAirflowCLICommandOperator` operator (#38965)
---
 .../providers/google/cloud/hooks/cloud_composer.py | 252 ++++++++++++++++++++-
 .../google/cloud/operators/cloud_composer.py       | 149 +++++++++++-
 .../google/cloud/triggers/cloud_composer.py        |  68 ++++++
 .../operators/cloud/cloud_composer.rst             |  20 ++
 tests/always/test_project_structure.py             |   1 -
 .../google/cloud/hooks/test_cloud_composer.py      | 128 +++++++++++
 .../google/cloud/operators/test_cloud_composer.py  |  66 +++++-
 .../google/cloud/triggers/test_cloud_composer.py   |  96 ++++++++
 .../cloud/composer/example_cloud_composer.py       |  25 ++
 9 files changed, 799 insertions(+), 6 deletions(-)

diff --git a/airflow/providers/google/cloud/hooks/cloud_composer.py 
b/airflow/providers/google/cloud/hooks/cloud_composer.py
index b5421e6b2b..4ee41c9ffd 100644
--- a/airflow/providers/google/cloud/hooks/cloud_composer.py
+++ b/airflow/providers/google/cloud/hooks/cloud_composer.py
@@ -17,7 +17,9 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Sequence
+import asyncio
+import time
+from typing import TYPE_CHECKING, MutableSequence, Sequence
 
 from google.api_core.client_options import ClientOptions
 from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
@@ -25,6 +27,7 @@ from google.cloud.orchestration.airflow.service_v1 import (
     EnvironmentsAsyncClient,
     EnvironmentsClient,
     ImageVersionsClient,
+    PollAirflowCommandResponse,
 )
 
 from airflow.exceptions import AirflowException
@@ -42,7 +45,10 @@ if TYPE_CHECKING:
     from 
google.cloud.orchestration.airflow.service_v1.services.image_versions.pagers 
import (
         ListImageVersionsPager,
     )
-    from google.cloud.orchestration.airflow.service_v1.types import Environment
+    from google.cloud.orchestration.airflow.service_v1.types import (
+        Environment,
+        ExecuteAirflowCommandResponse,
+    )
     from google.protobuf.field_mask_pb2 import FieldMask
 
 
@@ -294,6 +300,127 @@ class CloudComposerHook(GoogleBaseHook):
         )
         return result
 
+    @GoogleBaseHook.fallback_to_default_project_id
+    def execute_airflow_command(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        command: str,
+        subcommand: str,
+        parameters: MutableSequence[str],
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> ExecuteAirflowCommandResponse:
+        """
+        Execute Airflow command for provided Composer environment.
+
+        :param project_id: The ID of the Google Cloud project that the service 
belongs to.
+        :param region: The ID of the Google Cloud region that the service 
belongs to.
+        :param environment_id: The ID of the Google Cloud environment that the 
service belongs to.
+        :param command: Airflow command.
+        :param subcommand: Airflow subcommand.
+        :param parameters: Parameters for the Airflow command/subcommand as an 
array of arguments. It may
+            contain positional arguments like ``["my-dag-id"]``, key-value 
parameters like ``["--foo=bar"]``
+            or ``["--foo","bar"]``, or other flags like ``["-f"]``.
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request 
as metadata.
+        """
+        client = self.get_environment_client()
+        result = client.execute_airflow_command(
+            request={
+                "environment": self.get_environment_name(project_id, region, 
environment_id),
+                "command": command,
+                "subcommand": subcommand,
+                "parameters": parameters,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        return result
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    def poll_airflow_command(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        execution_id: str,
+        pod: str,
+        pod_namespace: str,
+        next_line_number: int,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> PollAirflowCommandResponse:
+        """
+        Poll Airflow command execution result for provided Composer 
environment.
+
+        :param project_id: The ID of the Google Cloud project that the service 
belongs to.
+        :param region: The ID of the Google Cloud region that the service 
belongs to.
+        :param environment_id: The ID of the Google Cloud environment that the 
service belongs to.
+        :param execution_id: The unique ID of the command execution.
+        :param pod: The name of the pod where the command is executed.
+        :param pod_namespace: The namespace of the pod where the command is 
executed.
+        :param next_line_number: Line number from which new logs should be 
fetched.
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request 
as metadata.
+        """
+        client = self.get_environment_client()
+        result = client.poll_airflow_command(
+            request={
+                "environment": self.get_environment_name(project_id, region, 
environment_id),
+                "execution_id": execution_id,
+                "pod": pod,
+                "pod_namespace": pod_namespace,
+                "next_line_number": next_line_number,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+        return result
+
+    def wait_command_execution_result(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        execution_cmd_info: dict,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+        poll_interval: int = 10,
+    ) -> dict:
+        while True:
+            try:
+                result = self.poll_airflow_command(
+                    project_id=project_id,
+                    region=region,
+                    environment_id=environment_id,
+                    execution_id=execution_cmd_info["execution_id"],
+                    pod=execution_cmd_info["pod"],
+                    pod_namespace=execution_cmd_info["pod_namespace"],
+                    next_line_number=1,
+                    retry=retry,
+                    timeout=timeout,
+                    metadata=metadata,
+                )
+            except Exception as ex:
+                self.log.exception("Exception occurred while polling CMD 
result")
+                raise AirflowException(ex)
+
+            result_dict = PollAirflowCommandResponse.to_dict(result)
+            if result_dict["output_end"]:
+                return result_dict
+
+            self.log.info("Waiting for result...")
+            time.sleep(poll_interval)
+
 
 class CloudComposerAsyncHook(GoogleBaseHook):
     """Hook for Google Cloud Composer async APIs."""
@@ -421,3 +548,124 @@ class CloudComposerAsyncHook(GoogleBaseHook):
             timeout=timeout,
             metadata=metadata,
         )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    async def execute_airflow_command(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        command: str,
+        subcommand: str,
+        parameters: MutableSequence[str],
+        retry: AsyncRetry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> AsyncOperation:
+        """
+        Execute Airflow command for provided Composer environment.
+
+        :param project_id: The ID of the Google Cloud project that the service 
belongs to.
+        :param region: The ID of the Google Cloud region that the service 
belongs to.
+        :param environment_id: The ID of the Google Cloud environment that the 
service belongs to.
+        :param command: Airflow command.
+        :param subcommand: Airflow subcommand.
+        :param parameters: Parameters for the Airflow command/subcommand as an 
array of arguments. It may
+            contain positional arguments like ``["my-dag-id"]``, key-value 
parameters like ``["--foo=bar"]``
+            or ``["--foo","bar"]``, or other flags like ``["-f"]``.
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request 
as metadata.
+        """
+        client = self.get_environment_client()
+
+        return await client.execute_airflow_command(
+            request={
+                "environment": self.get_environment_name(project_id, region, 
environment_id),
+                "command": command,
+                "subcommand": subcommand,
+                "parameters": parameters,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+
+    @GoogleBaseHook.fallback_to_default_project_id
+    async def poll_airflow_command(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        execution_id: str,
+        pod: str,
+        pod_namespace: str,
+        next_line_number: int,
+        retry: AsyncRetry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+    ) -> AsyncOperation:
+        """
+        Poll Airflow command execution result for provided Composer 
environment.
+
+        :param project_id: The ID of the Google Cloud project that the service 
belongs to.
+        :param region: The ID of the Google Cloud region that the service 
belongs to.
+        :param environment_id: The ID of the Google Cloud environment that the 
service belongs to.
+        :param execution_id: The unique ID of the command execution.
+        :param pod: The name of the pod where the command is executed.
+        :param pod_namespace: The namespace of the pod where the command is 
executed.
+        :param next_line_number: Line number from which new logs should be 
fetched.
+        :param retry: Designation of what errors, if any, should be retried.
+        :param timeout: The timeout for this request.
+        :param metadata: Strings which should be sent along with the request 
as metadata.
+        """
+        client = self.get_environment_client()
+
+        return await client.poll_airflow_command(
+            request={
+                "environment": self.get_environment_name(project_id, region, 
environment_id),
+                "execution_id": execution_id,
+                "pod": pod,
+                "pod_namespace": pod_namespace,
+                "next_line_number": next_line_number,
+            },
+            retry=retry,
+            timeout=timeout,
+            metadata=metadata,
+        )
+
+    async def wait_command_execution_result(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        execution_cmd_info: dict,
+        retry: AsyncRetry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+        poll_interval: int = 10,
+    ) -> dict:
+        while True:
+            try:
+                result = await self.poll_airflow_command(
+                    project_id=project_id,
+                    region=region,
+                    environment_id=environment_id,
+                    execution_id=execution_cmd_info["execution_id"],
+                    pod=execution_cmd_info["pod"],
+                    pod_namespace=execution_cmd_info["pod_namespace"],
+                    next_line_number=1,
+                    retry=retry,
+                    timeout=timeout,
+                    metadata=metadata,
+                )
+            except Exception as ex:
+                self.log.exception("Exception occurred while polling CMD 
result")
+                raise AirflowException(ex)
+
+            result_dict = PollAirflowCommandResponse.to_dict(result)
+            if result_dict["output_end"]:
+                return result_dict
+
+            self.log.info("Sleeping for %s seconds.", poll_interval)
+            await asyncio.sleep(poll_interval)
diff --git a/airflow/providers/google/cloud/operators/cloud_composer.py 
b/airflow/providers/google/cloud/operators/cloud_composer.py
index de6d49d656..ad63397a17 100644
--- a/airflow/providers/google/cloud/operators/cloud_composer.py
+++ b/airflow/providers/google/cloud/operators/cloud_composer.py
@@ -17,19 +17,23 @@
 # under the License.
 from __future__ import annotations
 
+import shlex
 from typing import TYPE_CHECKING, Sequence
 
 from google.api_core.exceptions import AlreadyExists
 from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
 from google.cloud.orchestration.airflow.service_v1 import ImageVersion
-from google.cloud.orchestration.airflow.service_v1.types import Environment
+from google.cloud.orchestration.airflow.service_v1.types import Environment, 
ExecuteAirflowCommandResponse
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.providers.google.cloud.hooks.cloud_composer import 
CloudComposerHook
 from airflow.providers.google.cloud.links.base import BaseGoogleLink
 from airflow.providers.google.cloud.operators.cloud_base import 
GoogleCloudBaseOperator
-from airflow.providers.google.cloud.triggers.cloud_composer import 
CloudComposerExecutionTrigger
+from airflow.providers.google.cloud.triggers.cloud_composer import (
+    CloudComposerAirflowCLICommandTrigger,
+    CloudComposerExecutionTrigger,
+)
 from airflow.providers.google.common.consts import 
GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
 
 if TYPE_CHECKING:
@@ -651,3 +655,144 @@ class 
CloudComposerListImageVersionsOperator(GoogleCloudBaseOperator):
             metadata=self.metadata,
         )
         return [ImageVersion.to_dict(image) for image in result]
+
+
+class CloudComposerRunAirflowCLICommandOperator(GoogleCloudBaseOperator):
+    """
+    Run Airflow command for provided Composer environment.
+
+    :param project_id: The ID of the Google Cloud project that the service 
belongs to.
+    :param region: The ID of the Google Cloud region that the service belongs 
to.
+    :param environment_id: The ID of the Google Cloud environment that the 
service belongs to.
+    :param command: Airflow command.
+    :param retry: Designation of what errors, if any, should be retried.
+    :param timeout: The timeout for this request.
+    :param metadata: Strings which should be sent along with the request as 
metadata.
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud 
Platform.
+    :param impersonation_chain: Optional service account to impersonate using 
short-term
+        credentials, or chained list of accounts required to get the 
access_token
+        of the last account in the list, which will be impersonated in the 
request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding 
identity, with first
+        account from the list granting this role to the originating account 
(templated).
+    :param deferrable: Run operator in the deferrable mode
+    :param poll_interval: Optional: Control the rate of the poll for the 
result of deferrable run.
+        By default, the trigger will poll every 10 seconds.
+    """
+
+    template_fields = (
+        "project_id",
+        "region",
+        "environment_id",
+        "command",
+        "impersonation_chain",
+    )
+
+    def __init__(
+        self,
+        *,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        command: str,
+        retry: Retry | _MethodDefault = DEFAULT,
+        timeout: float | None = None,
+        metadata: Sequence[tuple[str, str]] = (),
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", 
fallback=False),
+        poll_interval: int = 10,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.project_id = project_id
+        self.region = region
+        self.environment_id = environment_id
+        self.command = command
+        self.retry = retry
+        self.timeout = timeout
+        self.metadata = metadata
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self.deferrable = deferrable
+        self.poll_interval = poll_interval
+
+    def execute(self, context: Context):
+        hook = CloudComposerHook(
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.impersonation_chain,
+        )
+
+        self.log.info("Executing the command: [ airflow %s ]...", self.command)
+
+        cmd, subcommand, parameters = self._parse_cmd_to_args(self.command)
+        execution_cmd_info = hook.execute_airflow_command(
+            project_id=self.project_id,
+            region=self.region,
+            environment_id=self.environment_id,
+            command=cmd,
+            subcommand=subcommand,
+            parameters=parameters,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+        )
+        execution_cmd_info_dict = 
ExecuteAirflowCommandResponse.to_dict(execution_cmd_info)
+
+        self.log.info("Command has been started. execution_id=%s", 
execution_cmd_info_dict["execution_id"])
+
+        if self.deferrable:
+            self.defer(
+                trigger=CloudComposerAirflowCLICommandTrigger(
+                    project_id=self.project_id,
+                    region=self.region,
+                    environment_id=self.environment_id,
+                    execution_cmd_info=execution_cmd_info_dict,
+                    gcp_conn_id=self.gcp_conn_id,
+                    impersonation_chain=self.impersonation_chain,
+                    poll_interval=self.poll_interval,
+                ),
+                method_name=GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME,
+            )
+            return
+
+        result = hook.wait_command_execution_result(
+            project_id=self.project_id,
+            region=self.region,
+            environment_id=self.environment_id,
+            execution_cmd_info=execution_cmd_info_dict,
+            retry=self.retry,
+            timeout=self.timeout,
+            metadata=self.metadata,
+            poll_interval=self.poll_interval,
+        )
+        result_str = self._merge_cmd_output_result(result)
+        self.log.info("Command execution result:\n%s", result_str)
+        return result
+
+    def execute_complete(self, context: Context, event: dict) -> dict:
+        if event and event["status"] == "error":
+            raise AirflowException(event["message"])
+        result: dict = event["result"]
+        result_str = self._merge_cmd_output_result(result)
+        self.log.info("Command execution result:\n%s", result_str)
+        return result
+
+    def _parse_cmd_to_args(self, cmd: str) -> tuple:
+        """Parse user command to command, subcommand and parameters."""
+        cmd_dict = shlex.split(cmd)
+        if not cmd_dict:
+            raise AirflowException("The provided command is empty.")
+
+        command = cmd_dict[0] if len(cmd_dict) >= 1 else None
+        subcommand = cmd_dict[1] if len(cmd_dict) >= 2 else None
+        parameters = cmd_dict[2:] if len(cmd_dict) >= 3 else None
+
+        return command, subcommand, parameters
+
+    def _merge_cmd_output_result(self, result) -> str:
+        """Merge output to one string."""
+        result_str = "\n".join(line_dict["content"] for line_dict in 
result["output"])
+        return result_str
diff --git a/airflow/providers/google/cloud/triggers/cloud_composer.py 
b/airflow/providers/google/cloud/triggers/cloud_composer.py
index 4e52783a37..ac5a00c60f 100644
--- a/airflow/providers/google/cloud/triggers/cloud_composer.py
+++ b/airflow/providers/google/cloud/triggers/cloud_composer.py
@@ -78,3 +78,71 @@ class CloudComposerExecutionTrigger(BaseTrigger):
                 "operation_done": operation.done,
             }
         )
+
+
+class CloudComposerAirflowCLICommandTrigger(BaseTrigger):
+    """The trigger wait for the Airflow CLI command result."""
+
+    def __init__(
+        self,
+        project_id: str,
+        region: str,
+        environment_id: str,
+        execution_cmd_info: dict,
+        gcp_conn_id: str = "google_cloud_default",
+        impersonation_chain: str | Sequence[str] | None = None,
+        poll_interval: int = 10,
+    ):
+        super().__init__()
+        self.project_id = project_id
+        self.region = region
+        self.environment_id = environment_id
+        self.execution_cmd_info = execution_cmd_info
+        self.gcp_conn_id = gcp_conn_id
+        self.impersonation_chain = impersonation_chain
+        self.poll_interval = poll_interval
+
+        self.gcp_hook = CloudComposerAsyncHook(
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.impersonation_chain,
+        )
+
+    def serialize(self) -> tuple[str, dict[str, Any]]:
+        return (
+            
"airflow.providers.google.cloud.triggers.cloud_composer.CloudComposerAirflowCLICommandTrigger",
+            {
+                "project_id": self.project_id,
+                "region": self.region,
+                "environment_id": self.environment_id,
+                "execution_cmd_info": self.execution_cmd_info,
+                "gcp_conn_id": self.gcp_conn_id,
+                "impersonation_chain": self.impersonation_chain,
+                "poll_interval": self.poll_interval,
+            },
+        )
+
+    async def run(self):
+        try:
+            result = await self.gcp_hook.wait_command_execution_result(
+                project_id=self.project_id,
+                region=self.region,
+                environment_id=self.environment_id,
+                execution_cmd_info=self.execution_cmd_info,
+                poll_interval=self.poll_interval,
+            )
+        except AirflowException as ex:
+            yield TriggerEvent(
+                {
+                    "status": "error",
+                    "message": str(ex),
+                }
+            )
+            return
+
+        yield TriggerEvent(
+            {
+                "status": "success",
+                "result": result,
+            }
+        )
+        return
diff --git 
a/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst 
b/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst
index b55063c8f6..cdb9cb2931 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/cloud_composer.rst
@@ -157,3 +157,23 @@ You can also list all supported Cloud Composer images:
     :dedent: 4
     :start-after: [START howto_operator_composer_image_list]
     :end-before: [END howto_operator_composer_image_list]
+
+Run Airflow CLI commands
+------------------------
+
+You can run Airflow CLI commands in your environments, use:
+:class:`~airflow.providers.google.cloud.operators.cloud_composer.CloudComposerRunAirflowCLICommandOperator`
+
+.. exampleinclude:: 
/../../tests/system/providers/google/cloud/composer/example_cloud_composer.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_run_airflow_cli_command]
+    :end-before: [END howto_operator_run_airflow_cli_command]
+
+or you can define the same operator in the deferrable mode:
+
+.. exampleinclude:: 
/../../tests/system/providers/google/cloud/composer/example_cloud_composer.py
+    :language: python
+    :dedent: 4
+    :start-after: [START 
howto_operator_run_airflow_cli_command_deferrable_mode]
+    :end-before: [END howto_operator_run_airflow_cli_command_deferrable_mode]
diff --git a/tests/always/test_project_structure.py 
b/tests/always/test_project_structure.py
index 3437092e65..75d824732b 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -147,7 +147,6 @@ class TestProjectStructure:
             "tests/providers/google/cloud/transfers/test_mssql_to_gcs.py",
             "tests/providers/google/cloud/transfers/test_presto_to_gcs.py",
             "tests/providers/google/cloud/transfers/test_trino_to_gcs.py",
-            "tests/providers/google/cloud/triggers/test_cloud_composer.py",
             "tests/providers/google/cloud/utils/test_bigquery.py",
             "tests/providers/google/cloud/utils/test_bigquery_get_data.py",
             "tests/providers/google/cloud/utils/test_dataform.py",
diff --git a/tests/providers/google/cloud/hooks/test_cloud_composer.py 
b/tests/providers/google/cloud/hooks/test_cloud_composer.py
index 1072b310b2..945a637605 100644
--- a/tests/providers/google/cloud/hooks/test_cloud_composer.py
+++ b/tests/providers/google/cloud/hooks/test_cloud_composer.py
@@ -37,6 +37,12 @@ TEST_ENVIRONMENT = {
         "software_config": {"image_version": "composer-1.17.7-airflow-2.1.4"},
     },
 }
+TEST_COMMAND = "dags"
+TEST_SUBCOMMAND = "list"
+TEST_PARAMETERS = ["--verbose", "-o", "json"]
+TEST_EXECUTION_ID = "test-execution-id"
+TEST_POD = "test-pod"
+TEST_POD_NAMESPACE = "test-namespace"
 
 TEST_UPDATE_MASK = {"paths": ["labels.label1"]}
 TEST_UPDATED_ENVIRONMENT = {
@@ -197,6 +203,64 @@ class TestCloudComposerHook:
             metadata=TEST_METADATA,
         )
 
+    
@mock.patch(COMPOSER_STRING.format("CloudComposerHook.get_environment_client"))
+    def test_execute_airflow_command(self, mock_client) -> None:
+        self.hook.execute_airflow_command(
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            command=TEST_COMMAND,
+            subcommand=TEST_SUBCOMMAND,
+            parameters=TEST_PARAMETERS,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+        mock_client.assert_called_once()
+        
mock_client.return_value.execute_airflow_command.assert_called_once_with(
+            request={
+                "environment": self.hook.get_environment_name(
+                    TEST_GCP_PROJECT, TEST_GCP_REGION, TEST_ENVIRONMENT_ID
+                ),
+                "command": TEST_COMMAND,
+                "subcommand": TEST_SUBCOMMAND,
+                "parameters": TEST_PARAMETERS,
+            },
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+
+    
@mock.patch(COMPOSER_STRING.format("CloudComposerHook.get_environment_client"))
+    def test_poll_airflow_command(self, mock_client) -> None:
+        self.hook.poll_airflow_command(
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            execution_id=TEST_EXECUTION_ID,
+            pod=TEST_POD,
+            pod_namespace=TEST_POD_NAMESPACE,
+            next_line_number=1,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+        mock_client.assert_called_once()
+        mock_client.return_value.poll_airflow_command.assert_called_once_with(
+            request={
+                "environment": self.hook.get_environment_name(
+                    TEST_GCP_PROJECT, TEST_GCP_REGION, TEST_ENVIRONMENT_ID
+                ),
+                "execution_id": TEST_EXECUTION_ID,
+                "pod": TEST_POD,
+                "pod_namespace": TEST_POD_NAMESPACE,
+                "next_line_number": 1,
+            },
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+
 
 class TestCloudComposerAsyncHook:
     def test_delegate_to_runtime_error(self):
@@ -282,3 +346,67 @@ class TestCloudComposerAsyncHook:
             timeout=TEST_TIMEOUT,
             metadata=TEST_METADATA,
         )
+
+    @pytest.mark.asyncio
+    
@mock.patch(COMPOSER_STRING.format("CloudComposerAsyncHook.get_environment_client"))
+    async def test_execute_airflow_command(self, mock_client) -> None:
+        mock_env_client = AsyncMock(EnvironmentsAsyncClient)
+        mock_client.return_value = mock_env_client
+        await self.hook.execute_airflow_command(
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            command=TEST_COMMAND,
+            subcommand=TEST_SUBCOMMAND,
+            parameters=TEST_PARAMETERS,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+        mock_client.assert_called_once()
+        
mock_client.return_value.execute_airflow_command.assert_called_once_with(
+            request={
+                "environment": self.hook.get_environment_name(
+                    TEST_GCP_PROJECT, TEST_GCP_REGION, TEST_ENVIRONMENT_ID
+                ),
+                "command": TEST_COMMAND,
+                "subcommand": TEST_SUBCOMMAND,
+                "parameters": TEST_PARAMETERS,
+            },
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+
+    @pytest.mark.asyncio
+    
@mock.patch(COMPOSER_STRING.format("CloudComposerAsyncHook.get_environment_client"))
+    async def test_poll_airflow_command(self, mock_client) -> None:
+        mock_env_client = AsyncMock(EnvironmentsAsyncClient)
+        mock_client.return_value = mock_env_client
+        await self.hook.poll_airflow_command(
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            execution_id=TEST_EXECUTION_ID,
+            pod=TEST_POD,
+            pod_namespace=TEST_POD_NAMESPACE,
+            next_line_number=1,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+        mock_client.assert_called_once()
+        mock_client.return_value.poll_airflow_command.assert_called_once_with(
+            request={
+                "environment": self.hook.get_environment_name(
+                    TEST_GCP_PROJECT, TEST_GCP_REGION, TEST_ENVIRONMENT_ID
+                ),
+                "execution_id": TEST_EXECUTION_ID,
+                "pod": TEST_POD,
+                "pod_namespace": TEST_POD_NAMESPACE,
+                "next_line_number": 1,
+            },
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
diff --git a/tests/providers/google/cloud/operators/test_cloud_composer.py 
b/tests/providers/google/cloud/operators/test_cloud_composer.py
index 7bda6fce5a..e882db9526 100644
--- a/tests/providers/google/cloud/operators/test_cloud_composer.py
+++ b/tests/providers/google/cloud/operators/test_cloud_composer.py
@@ -28,9 +28,13 @@ from airflow.providers.google.cloud.operators.cloud_composer 
import (
     CloudComposerGetEnvironmentOperator,
     CloudComposerListEnvironmentsOperator,
     CloudComposerListImageVersionsOperator,
+    CloudComposerRunAirflowCLICommandOperator,
     CloudComposerUpdateEnvironmentOperator,
 )
-from airflow.providers.google.cloud.triggers.cloud_composer import 
CloudComposerExecutionTrigger
+from airflow.providers.google.cloud.triggers.cloud_composer import (
+    CloudComposerAirflowCLICommandTrigger,
+    CloudComposerExecutionTrigger,
+)
 from airflow.providers.google.common.consts import 
GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
 
 TASK_ID = "task-id"
@@ -46,6 +50,10 @@ TEST_ENVIRONMENT = {
         "software_config": {"image_version": "composer-1.17.7-airflow-2.1.4"},
     },
 }
+TEST_USER_COMMAND = "dags list -o json --verbose"
+TEST_COMMAND = "dags"
+TEST_SUBCOMMAND = "list"
+TEST_PARAMETERS = ["-o", "json", "--verbose"]
 
 TEST_UPDATE_MASK = {"paths": ["labels.label1"]}
 TEST_UPDATED_ENVIRONMENT = {
@@ -305,3 +313,59 @@ class TestCloudComposerListImageVersionsOperator:
             timeout=TEST_TIMEOUT,
             metadata=TEST_METADATA,
         )
+
+
+class TestCloudComposerRunAirflowCLICommandOperator:
+    
@mock.patch(COMPOSER_STRING.format("ExecuteAirflowCommandResponse.to_dict"))
+    @mock.patch(COMPOSER_STRING.format("CloudComposerHook"))
+    def test_execute(self, mock_hook, to_dict_mode) -> None:
+        op = CloudComposerRunAirflowCLICommandOperator(
+            task_id=TASK_ID,
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            command=TEST_USER_COMMAND,
+            gcp_conn_id=TEST_GCP_CONN_ID,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+        op.execute(mock.MagicMock())
+        mock_hook.assert_called_once_with(
+            gcp_conn_id=TEST_GCP_CONN_ID,
+            impersonation_chain=TEST_IMPERSONATION_CHAIN,
+        )
+        mock_hook.return_value.execute_airflow_command.assert_called_once_with(
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            command=TEST_COMMAND,
+            subcommand=TEST_SUBCOMMAND,
+            parameters=TEST_PARAMETERS,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+        )
+
+    
@mock.patch(COMPOSER_STRING.format("ExecuteAirflowCommandResponse.to_dict"))
+    @mock.patch(COMPOSER_STRING.format("CloudComposerHook"))
+    @mock.patch(COMPOSER_TRIGGERS_STRING.format("CloudComposerAsyncHook"))
+    def test_execute_deferrable(self, mock_trigger_hook, mock_hook, 
to_dict_mode):
+        op = CloudComposerRunAirflowCLICommandOperator(
+            task_id=TASK_ID,
+            project_id=TEST_GCP_PROJECT,
+            region=TEST_GCP_REGION,
+            environment_id=TEST_ENVIRONMENT_ID,
+            command=TEST_USER_COMMAND,
+            gcp_conn_id=TEST_GCP_CONN_ID,
+            retry=TEST_RETRY,
+            timeout=TEST_TIMEOUT,
+            metadata=TEST_METADATA,
+            deferrable=True,
+        )
+
+        with pytest.raises(TaskDeferred) as exc:
+            op.execute(mock.MagicMock())
+
+        assert isinstance(exc.value.trigger, 
CloudComposerAirflowCLICommandTrigger)
+        assert exc.value.method_name == GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
diff --git a/tests/providers/google/cloud/triggers/test_cloud_composer.py 
b/tests/providers/google/cloud/triggers/test_cloud_composer.py
new file mode 100644
index 0000000000..99daaf83bd
--- /dev/null
+++ b/tests/providers/google/cloud/triggers/test_cloud_composer.py
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from unittest import mock
+
+import pytest
+
+from airflow.models import Connection
+from airflow.providers.google.cloud.triggers.cloud_composer import 
CloudComposerAirflowCLICommandTrigger
+from airflow.triggers.base import TriggerEvent
+
+TEST_PROJECT_ID = "test-project-id"
+TEST_LOCATION = "us-central1"
+TEST_ENVIRONMENT_ID = "testenvname"
+TEST_EXEC_CMD_INFO = {
+    "execution_id": "test_id",
+    "pod": "test_pod",
+    "pod_namespace": "test_namespace",
+    "error": "test_error",
+}
+TEST_GCP_CONN_ID = "test_gcp_conn_id"
+TEST_POLL_INTERVAL = 10
+TEST_IMPERSONATION_CHAIN = "test_impersonation_chain"
+TEST_EXEC_RESULT = {
+    "output": [{"line_number": 1, "content": "test_content"}],
+    "output_end": True,
+    "exit_info": {"exit_code": 0, "error": ""},
+}
+
+
[email protected]
[email protected](
+    
"airflow.providers.google.common.hooks.base_google.GoogleBaseHook.get_connection",
+    return_value=Connection(conn_id="test_conn"),
+)
+def trigger(mock_conn):
+    return CloudComposerAirflowCLICommandTrigger(
+        project_id=TEST_PROJECT_ID,
+        region=TEST_LOCATION,
+        environment_id=TEST_ENVIRONMENT_ID,
+        execution_cmd_info=TEST_EXEC_CMD_INFO,
+        gcp_conn_id=TEST_GCP_CONN_ID,
+        impersonation_chain=TEST_IMPERSONATION_CHAIN,
+        poll_interval=TEST_POLL_INTERVAL,
+    )
+
+
+class TestCloudComposerAirflowCLICommandTrigger:
+    def test_serialize(self, trigger):
+        actual_data = trigger.serialize()
+        expected_data = (
+            
"airflow.providers.google.cloud.triggers.cloud_composer.CloudComposerAirflowCLICommandTrigger",
+            {
+                "project_id": TEST_PROJECT_ID,
+                "region": TEST_LOCATION,
+                "environment_id": TEST_ENVIRONMENT_ID,
+                "execution_cmd_info": TEST_EXEC_CMD_INFO,
+                "gcp_conn_id": TEST_GCP_CONN_ID,
+                "impersonation_chain": TEST_IMPERSONATION_CHAIN,
+                "poll_interval": TEST_POLL_INTERVAL,
+            },
+        )
+        assert actual_data == expected_data
+
+    @pytest.mark.asyncio
+    @mock.patch(
+        
"airflow.providers.google.cloud.hooks.cloud_composer.CloudComposerAsyncHook.wait_command_execution_result"
+    )
+    async def test_run(self, mock_exec_result, trigger):
+        mock_exec_result.return_value = TEST_EXEC_RESULT
+
+        expected_event = TriggerEvent(
+            {
+                "status": "success",
+                "result": TEST_EXEC_RESULT,
+            }
+        )
+        actual_event = await trigger.run().asend(None)
+
+        assert actual_event == expected_event
diff --git 
a/tests/system/providers/google/cloud/composer/example_cloud_composer.py 
b/tests/system/providers/google/cloud/composer/example_cloud_composer.py
index fb8958412e..fe60c56ddf 100644
--- a/tests/system/providers/google/cloud/composer/example_cloud_composer.py
+++ b/tests/system/providers/google/cloud/composer/example_cloud_composer.py
@@ -28,6 +28,7 @@ from airflow.providers.google.cloud.operators.cloud_composer 
import (
     CloudComposerGetEnvironmentOperator,
     CloudComposerListEnvironmentsOperator,
     CloudComposerListImageVersionsOperator,
+    CloudComposerRunAirflowCLICommandOperator,
     CloudComposerUpdateEnvironmentOperator,
 )
 from airflow.utils.trigger_rule import TriggerRule
@@ -59,6 +60,8 @@ UPDATED_ENVIRONMENT = {
 UPDATE_MASK = {"paths": ["labels.label"]}
 # [END howto_operator_composer_update_environment]
 
+COMMAND = "dags list -o json --verbose"
+
 
 with DAG(
     DAG_ID,
@@ -134,6 +137,27 @@ with DAG(
     )
     # [END howto_operator_update_composer_environment_deferrable_mode]
 
+    # [START howto_operator_run_airflow_cli_command]
+    run_airflow_cli_cmd = CloudComposerRunAirflowCLICommandOperator(
+        task_id="run_airflow_cli_cmd",
+        project_id=PROJECT_ID,
+        region=REGION,
+        environment_id=ENVIRONMENT_ID,
+        command=COMMAND,
+    )
+    # [END howto_operator_run_airflow_cli_command]
+
+    # [START howto_operator_run_airflow_cli_command_deferrable_mode]
+    defer_run_airflow_cli_cmd = CloudComposerRunAirflowCLICommandOperator(
+        task_id="defer_run_airflow_cli_cmd",
+        project_id=PROJECT_ID,
+        region=REGION,
+        environment_id=ENVIRONMENT_ID_ASYNC,
+        command=COMMAND,
+        deferrable=True,
+    )
+    # [END howto_operator_run_airflow_cli_command_deferrable_mode]
+
     # [START howto_operator_delete_composer_environment]
     delete_env = CloudComposerDeleteEnvironmentOperator(
         task_id="delete_env",
@@ -161,6 +185,7 @@ with DAG(
         list_envs,
         get_env,
         [update_env, defer_update_env],
+        [run_airflow_cli_cmd, defer_run_airflow_cli_cmd],
         [delete_env, defer_delete_env],
     )
 


Reply via email to