This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c17140cb37 Fix Storage Transfer system tests (#33385)
c17140cb37 is described below
commit c17140cb379b461ebe2019f1ea4289d639298a55
Author: VladaZakharova <[email protected]>
AuthorDate: Mon Aug 14 12:55:00 2023 +0200
Fix Storage Transfer system tests (#33385)
---
.../cloud/cloud_storage_transfer_service.rst | 2 +-
.../example_cloud_storage_transfer_service_gcp.py | 42 ++++++++--------------
...e_cloud_storage_transfer_service_gcs_to_gcs.py} | 20 ++++-------
.../resources/{file => transfer_service_gcp_file} | 0
.../{file => transfer_service_gcs_to_gcs_file} | 0
5 files changed, 22 insertions(+), 42 deletions(-)
diff --git
a/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
b/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
index cdeb0ea71e..4c7309cf4c 100644
---
a/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
+++
b/docs/apache-airflow-providers-google/operators/cloud/cloud_storage_transfer_service.rst
@@ -383,7 +383,7 @@ For parameter definition, take a look at
Using the operator
""""""""""""""""""
-.. exampleinclude::
/../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp_to_gcs.py
+.. exampleinclude::
/../../tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py
:language: python
:dedent: 4
:start-after: [START howto_operator_transfer_gcs_to_gcs]
diff --git
a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
b/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
index c207cfd715..3bf1f43a20 100644
---
a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
+++
b/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp.py
@@ -18,14 +18,6 @@
"""
Example Airflow DAG that demonstrates interactions with Google Cloud Transfer.
-
-
-This DAG relies on the following OS environment variables
-
-* GCP_PROJECT_ID - Google Cloud Project to use for the Google Cloud Transfer
Service.
-* GCP_TRANSFER_FIRST_TARGET_BUCKET - Google Cloud Storage bucket to which
files are copied from AWS.
- It is also a source bucket in next step
-* GCP_TRANSFER_SECOND_TARGET_BUCKET - Google Cloud Storage bucket to which
files are copied
"""
from __future__ import annotations
@@ -35,7 +27,6 @@ from datetime import datetime, timedelta
from pathlib import Path
from airflow import models
-from airflow.models.baseoperator import chain
from airflow.providers.google.cloud.hooks.cloud_storage_transfer_service
import (
ALREADY_EXISTING_IN_SINK,
BUCKET_NAME,
@@ -71,14 +62,14 @@ from
airflow.providers.google.cloud.sensors.cloud_storage_transfer_service impor
from airflow.providers.google.cloud.transfers.local_to_gcs import
LocalFilesystemToGCSOperator
from airflow.utils.trigger_rule import TriggerRule
-ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
-PROJECT_ID_TRANSFER = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
+PROJECT_ID_TRANSFER = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
DAG_ID = "example_gcp_transfer"
-BUCKET_NAME_SRC = f"src-bucket-{DAG_ID}-{ENV_ID}"
-BUCKET_NAME_DST = f"dst-bucket-{DAG_ID}-{ENV_ID}"
-FILE_NAME = "file"
+BUCKET_NAME_SRC = f"src-bucket-{DAG_ID}-{ENV_ID}".replace("_", "-")
+BUCKET_NAME_DST = f"dst-bucket-{DAG_ID}-{ENV_ID}".replace("_", "-")
+FILE_NAME = "transfer_service_gcp_file"
FILE_URI = f"gs://{BUCKET_NAME_SRC}/{FILE_NAME}"
CURRENT_FOLDER = Path(__file__).parent
@@ -115,7 +106,7 @@ with models.DAG(
schedule="@once", # Override to match your needs
start_date=datetime(2021, 1, 1),
catchup=False,
- tags=["example"],
+ tags=["example", "transfer", "gcp"],
) as dag:
create_bucket_src = GCSCreateBucketOperator(
@@ -184,18 +175,15 @@ with models.DAG(
task_id="delete_bucket_src", bucket_name=BUCKET_NAME_SRC,
trigger_rule=TriggerRule.ALL_DONE
)
- chain(
- create_bucket_src,
- upload_file,
- create_bucket_dst,
- create_transfer,
- wait_for_transfer,
- update_transfer,
- list_operations,
- get_operation,
- delete_transfer,
- delete_bucket_src,
- delete_bucket_dst,
+ (
+ [create_bucket_src, create_bucket_dst]
+ >> upload_file
+ >> create_transfer
+ >> wait_for_transfer
+ >> update_transfer
+ >> list_operations
+ >> get_operation
+ >> [delete_transfer, delete_bucket_src, delete_bucket_dst]
)
from tests.system.utils.watcher import watcher
diff --git
a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp_to_gcs.py
b/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py
similarity index 84%
rename from
tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp_to_gcs.py
rename to
tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py
index 01a6abeb95..93f7547a0b 100644
---
a/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp_to_gcs.py
+++
b/tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcs_to_gcs.py
@@ -18,14 +18,6 @@
"""
Example Airflow DAG that demonstrates interactions with Google Cloud Transfer.
-
-
-This DAG relies on the following OS environment variables
-
-* GCP_PROJECT_ID - Google Cloud Project to use for the Google Cloud Transfer
Service.
-* GCP_TRANSFER_FIRST_TARGET_BUCKET - Google Cloud Storage bucket to which
files are copied from AWS.
- It is also a source bucket in next step
-* GCP_TRANSFER_SECOND_TARGET_BUCKET - Google Cloud Storage bucket to which
files are copied
"""
from __future__ import annotations
@@ -45,13 +37,13 @@ from airflow.providers.google.cloud.operators.gcs import (
from airflow.providers.google.cloud.transfers.local_to_gcs import
LocalFilesystemToGCSOperator
from airflow.utils.trigger_rule import TriggerRule
-ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
-PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID", "default")
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default")
DAG_ID = "example_transfer_gcs_to_gcs"
-BUCKET_NAME_SRC = f"src-bucket-{DAG_ID}-{ENV_ID}"
-BUCKET_NAME_DST = f"dst-bucket-{DAG_ID}-{ENV_ID}"
-FILE_NAME = "file"
+BUCKET_NAME_SRC = f"src-bucket-{DAG_ID}-{ENV_ID}".replace("_", "-")
+BUCKET_NAME_DST = f"dst-bucket-{DAG_ID}-{ENV_ID}".replace("_", "-")
+FILE_NAME = "transfer_service_gcs_to_gcs_file"
FILE_URI = f"gs://{BUCKET_NAME_SRC}/{FILE_NAME}"
CURRENT_FOLDER = Path(__file__).parent
@@ -63,7 +55,7 @@ with models.DAG(
schedule="@once", # Override to match your needs
start_date=datetime(2021, 1, 1),
catchup=False,
- tags=["example", "transfer", "gcs"],
+ tags=["example", "transfer", "gcs-to-gcs"],
) as dag:
create_bucket_src = GCSCreateBucketOperator(
diff --git
a/tests/system/providers/google/cloud/storage_transfer/resources/file
b/tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcp_file
similarity index 100%
copy from tests/system/providers/google/cloud/storage_transfer/resources/file
copy to
tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcp_file
diff --git
a/tests/system/providers/google/cloud/storage_transfer/resources/file
b/tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file
similarity index 100%
rename from tests/system/providers/google/cloud/storage_transfer/resources/file
rename to
tests/system/providers/google/cloud/storage_transfer/resources/transfer_service_gcs_to_gcs_file
