This is an automated email from the ASF dual-hosted git repository.

joshfell pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 70121f82f7 Implement `CloudDataTransferServiceRunJobOperator` (#39154)
70121f82f7 is described below

commit 70121f82f74ef3866f97fc22420dcb806fd94126
Author: Bora Berke Sahin <[email protected]>
AuthorDate: Tue May 28 22:01:53 2024 +0300

    Implement `CloudDataTransferServiceRunJobOperator` (#39154)
    
    * Implement CloudDataTransferServiceRunJobOperator
    
    * Add references from storage-transfer docs
    
    * Add unit test for `run_transfer_job`
    
    * Add docs and example dag usage for 
`CloudDataTransferServiceRunJobOperator`
    
    * Fix doctest errors
    
    * Validate inputs inside execute function
    
    * Remove validation check in the constructor
    
    * Fix failing test
    
    * Fix ruff linter issues
    
    * Ensure consistent `project_id` usage
    
    Co-authored-by: Josh Fell <[email protected]>
    
    ---------
    
    Co-authored-by: Josh Fell <[email protected]>
---
 .../cloud/hooks/cloud_storage_transfer_service.py  | 26 ++++++++
 .../operators/cloud_storage_transfer_service.py    | 76 ++++++++++++++++++++++
 .../cloud/cloud_storage_transfer_service.rst       | 35 ++++++++++
 .../hooks/test_cloud_storage_transfer_service.py   | 19 ++++++
 .../test_cloud_storage_transfer_service.py         | 56 ++++++++++++++++
 .../example_cloud_storage_transfer_service_gcp.py  | 10 +++
 6 files changed, 222 insertions(+)

diff --git 
a/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py 
b/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py
index 89d83c76ee..24db7eb4d9 100644
--- a/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py
+++ b/airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py
@@ -344,6 +344,32 @@ class CloudDataTransferServiceHook(GoogleBaseHook):
             .execute(num_retries=self.num_retries)
         )
 
+    @GoogleBaseHook.fallback_to_default_project_id
+    def run_transfer_job(self, job_name: str, project_id: str) -> dict:
+        """Run Google Storage Transfer Service job.
+
+        :param job_name: (Required) Name of the job to be fetched
+        :param project_id: (Optional) the ID of the project that owns the 
Transfer
+            Job. If set to None or missing, the default project_id from the 
Google Cloud
+            connection is used.
+        :return: If successful, Operation. See:
+            
https://cloud.google.com/storage-transfer/docs/reference/rest/v1/Operation
+
+        .. seealso:: 
https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/run
+
+        """
+        return (
+            self.get_conn()
+            .transferJobs()
+            .run(
+                jobName=job_name,
+                body={
+                    PROJECT_ID: project_id,
+                },
+            )
+            .execute(num_retries=self.num_retries)
+        )
+
     def cancel_transfer_operation(self, operation_name: str) -> None:
         """Cancel a transfer operation in Google Storage Transfer Service.
 
diff --git 
a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py 
b/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
index 81f4cccebf..20f1f5e029 100644
--- a/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
+++ b/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
@@ -443,6 +443,82 @@ class 
CloudDataTransferServiceDeleteJobOperator(GoogleCloudBaseOperator):
         hook.delete_transfer_job(job_name=self.job_name, 
project_id=self.project_id)
 
 
+class CloudDataTransferServiceRunJobOperator(GoogleCloudBaseOperator):
+    """
+    Runs a transfer job.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the 
guide:
+        :ref:`howto/operator:CloudDataTransferServiceRunJobOperator`
+
+    :param job_name: (Required) Name of the job to be run
+    :param project_id: (Optional) the ID of the project that owns the Transfer
+        Job. If set to None or missing, the default project_id from the Google 
Cloud
+        connection is used.
+    :param gcp_conn_id: The connection ID used to connect to Google Cloud.
+    :param api_version: API version used (e.g. v1).
+    :param google_impersonation_chain: Optional Google service account to 
impersonate using
+        short-term credentials, or chained list of accounts required to get 
the access_token
+        of the last account in the list, which will be impersonated in the 
request.
+        If set as a string, the account must grant the originating account
+        the Service Account Token Creator IAM role.
+        If set as a sequence, the identities from the list must grant
+        Service Account Token Creator IAM role to the directly preceding 
identity, with first
+        account from the list granting this role to the originating account 
(templated).
+    """
+
+    # [START gcp_transfer_job_run_template_fields]
+    template_fields: Sequence[str] = (
+        "job_name",
+        "project_id",
+        "gcp_conn_id",
+        "api_version",
+        "google_impersonation_chain",
+    )
+    # [END gcp_transfer_job_run_template_fields]
+    operator_extra_links = (CloudStorageTransferJobLink(),)
+
+    def __init__(
+        self,
+        *,
+        job_name: str,
+        gcp_conn_id: str = "google_cloud_default",
+        api_version: str = "v1",
+        project_id: str = PROVIDE_PROJECT_ID,
+        google_impersonation_chain: str | Sequence[str] | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.job_name = job_name
+        self.project_id = project_id
+        self.gcp_conn_id = gcp_conn_id
+        self.api_version = api_version
+        self.google_impersonation_chain = google_impersonation_chain
+
+    def _validate_inputs(self) -> None:
+        if not self.job_name:
+            raise AirflowException("The required parameter 'job_name' is empty 
or None")
+
+    def execute(self, context: Context) -> dict:
+        self._validate_inputs()
+        hook = CloudDataTransferServiceHook(
+            api_version=self.api_version,
+            gcp_conn_id=self.gcp_conn_id,
+            impersonation_chain=self.google_impersonation_chain,
+        )
+
+        project_id = self.project_id or hook.project_id
+        if project_id:
+            CloudStorageTransferJobLink.persist(
+                context=context,
+                task_instance=self,
+                project_id=project_id,
+                job_name=self.job_name,
+            )
+
+        return hook.run_transfer_job(job_name=self.job_name, 
project_id=project_id)
+
+
 class CloudDataTransferServiceGetOperationOperator(GoogleCloudBaseOperator):
     """
     Gets the latest state of a long-running operation in Google Storage 
Transfer Service.
diff --git 
a/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
 
b/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
index bd6a3fde3c..df3365631f 100644
--- 
a/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
+++ 
b/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
@@ -130,6 +130,41 @@ See `Google Cloud Transfer Service - REST Resource: 
transferJobs - Status
 
 .. _howto/operator:CloudDataTransferServiceUpdateJobOperator:
 
+CloudDataTransferServiceRunJobOperator
+-----------------------------------------
+
+Runs a transfer job.
+
+For parameter definition, take a look at
+:class:`~airflow.providers.google.cloud.operators.cloud_storage_transfer_service.CloudDataTransferServiceRunJobOperator`.
+
+
+Using the operator
+""""""""""""""""""
+
+.. exampleinclude:: 
/../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
+      :language: python
+      :dedent: 4
+      :start-after: [START howto_operator_gcp_transfer_run_job]
+      :end-before: [END howto_operator_gcp_transfer_run_job]
+
+Templating
+""""""""""
+
+.. literalinclude:: 
/../../airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py
+    :language: python
+    :dedent: 4
+    :start-after: [START gcp_transfer_job_run_template_fields]
+    :end-before: [END gcp_transfer_job_run_template_fields]
+
+More information
+""""""""""""""""
+
+See `Google Cloud Transfer Service - REST Resource: transferJobs - Run
+<https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs/run>`_
+
+.. _howto/operator:CloudDataTransferServiceRunJobOperator:
+
 CloudDataTransferServiceUpdateJobOperator
 -----------------------------------------
 
diff --git 
a/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py 
b/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py
index 3b41355cda..672a80d2fa 100644
--- a/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py
+++ b/tests/providers/google/cloud/hooks/test_cloud_storage_transfer_service.py
@@ -287,6 +287,25 @@ class TestGCPTransferServiceHookWithPassedProjectId:
         )
         execute_method.assert_called_once_with(num_retries=5)
 
+    @mock.patch(
+        "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service."
+        "CloudDataTransferServiceHook.get_conn"
+    )
+    def test_run_transfer_job(self, get_conn):
+        run_method = get_conn.return_value.transferJobs.return_value.run
+        execute_method = run_method.return_value.execute
+        execute_method.return_value = TEST_TRANSFER_OPERATION
+
+        res = self.gct_hook.run_transfer_job(job_name=TEST_TRANSFER_JOB_NAME, 
project_id=TEST_PROJECT_ID)
+        assert res == TEST_TRANSFER_OPERATION
+        run_method.assert_called_once_with(
+            jobName=TEST_TRANSFER_JOB_NAME,
+            body={
+                PROJECT_ID: TEST_PROJECT_ID,
+            },
+        )
+        execute_method.assert_called_once_with(num_retries=5)
+
     @mock.patch(
         "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service"
         ".CloudDataTransferServiceHook.get_conn"
diff --git 
a/tests/providers/google/cloud/operators/test_cloud_storage_transfer_service.py 
b/tests/providers/google/cloud/operators/test_cloud_storage_transfer_service.py
index d242a7e956..6b6106e47a 100644
--- 
a/tests/providers/google/cloud/operators/test_cloud_storage_transfer_service.py
+++ 
b/tests/providers/google/cloud/operators/test_cloud_storage_transfer_service.py
@@ -56,6 +56,7 @@ from 
airflow.providers.google.cloud.operators.cloud_storage_transfer_service imp
     CloudDataTransferServiceListOperationsOperator,
     CloudDataTransferServicePauseOperationOperator,
     CloudDataTransferServiceResumeOperationOperator,
+    CloudDataTransferServiceRunJobOperator,
     CloudDataTransferServiceS3ToGCSOperator,
     CloudDataTransferServiceUpdateJobOperator,
     TransferJobPreprocessor,
@@ -493,6 +494,61 @@ class TestGcpStorageTransferJobDeleteOperator:
             CloudDataTransferServiceDeleteJobOperator(job_name="", 
task_id="task-id")
 
 
+class TestGcpStorageTransferJobRunOperator:
+    @mock.patch(
+        
"airflow.providers.google.cloud.operators.cloud_storage_transfer_service.CloudDataTransferServiceHook"
+    )
+    def test_job_run(self, mock_hook):
+        mock_hook.return_value.run_transfer_job.return_value = VALID_OPERATION
+        op = CloudDataTransferServiceRunJobOperator(
+            job_name=JOB_NAME,
+            project_id=GCP_PROJECT_ID,
+            task_id="task-id",
+            google_impersonation_chain=IMPERSONATION_CHAIN,
+        )
+        result = op.execute(context=mock.MagicMock())
+        mock_hook.assert_called_once_with(
+            api_version="v1",
+            gcp_conn_id="google_cloud_default",
+            impersonation_chain=IMPERSONATION_CHAIN,
+        )
+        mock_hook.return_value.run_transfer_job.assert_called_once_with(
+            job_name=JOB_NAME, project_id=GCP_PROJECT_ID
+        )
+        assert result == VALID_OPERATION
+
+    # Setting all the operator's input parameters as templated dag_ids
+    # (could be anything else) just to test if the templating works for all
+    # fields
+    @pytest.mark.db_test
+    @mock.patch(
+        
"airflow.providers.google.cloud.operators.cloud_storage_transfer_service.CloudDataTransferServiceHook"
+    )
+    def test_job_run_with_templates(self, _, create_task_instance_of_operator):
+        dag_id = "test_job_run_with_templates"
+        ti = create_task_instance_of_operator(
+            CloudDataTransferServiceRunJobOperator,
+            dag_id=dag_id,
+            job_name="{{ dag.dag_id }}",
+            project_id="{{ dag.dag_id }}",
+            gcp_conn_id="{{ dag.dag_id }}",
+            api_version="{{ dag.dag_id }}",
+            google_impersonation_chain="{{ dag.dag_id }}",
+            task_id=TASK_ID,
+        )
+        ti.render_templates()
+        assert dag_id == ti.task.job_name
+        assert dag_id == ti.task.project_id
+        assert dag_id == ti.task.gcp_conn_id
+        assert dag_id == ti.task.api_version
+        assert dag_id == ti.task.google_impersonation_chain
+
+    def test_job_run_should_throw_ex_when_name_none(self):
+        op = CloudDataTransferServiceRunJobOperator(job_name="", 
task_id="task-id")
+        with pytest.raises(AirflowException, match="The required parameter 
'job_name' is empty or None"):
+            op.execute(context=mock.MagicMock())
+
+
 class TestGpcStorageTransferOperationsGetOperator:
     @mock.patch(
         
"airflow.providers.google.cloud.operators.cloud_storage_transfer_service.CloudDataTransferServiceHook"
diff --git 
a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
 
b/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
index b173abfffa..2920409ce7 100644
--- 
a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
+++ 
b/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
@@ -53,6 +53,7 @@ from 
airflow.providers.google.cloud.operators.cloud_storage_transfer_service imp
     CloudDataTransferServiceDeleteJobOperator,
     CloudDataTransferServiceGetOperationOperator,
     CloudDataTransferServiceListOperationsOperator,
+    CloudDataTransferServiceRunJobOperator,
     CloudDataTransferServiceUpdateJobOperator,
 )
 from airflow.providers.google.cloud.operators.gcs import 
GCSCreateBucketOperator, GCSDeleteBucketOperator
@@ -147,6 +148,14 @@ with DAG(
         expected_statuses={GcpTransferOperationStatus.SUCCESS},
     )
 
+    # [START howto_operator_gcp_transfer_run_job]
+    run_transfer = CloudDataTransferServiceRunJobOperator(
+        task_id="run_transfer",
+        job_name="{{task_instance.xcom_pull('create_transfer')['name']}}",
+        project_id=PROJECT_ID_TRANSFER,
+    )
+    # [END howto_operator_gcp_transfer_run_job]
+
     list_operations = CloudDataTransferServiceListOperationsOperator(
         task_id="list_operations",
         request_filter={
@@ -180,6 +189,7 @@ with DAG(
         >> create_transfer
         >> wait_for_transfer
         >> update_transfer
+        >> run_transfer
         >> list_operations
         >> get_operation
         >> [delete_transfer, delete_bucket_src, delete_bucket_dst]

Reply via email to