ferruzzi commented on code in PR #34999:
URL: https://github.com/apache/airflow/pull/34999#discussion_r1362513408
##########
tests/system/providers/amazon/aws/example_glue_databrew.py:
##########
@@ -16,31 +16,143 @@
# under the License.
from __future__ import annotations
+import boto3
import pendulum
+from airflow.decorators import task
from airflow.models.baseoperator import chain
from airflow.models.dag import DAG
from airflow.providers.amazon.aws.operators.glue_databrew import (
GlueDataBrewStartJobOperator,
)
+from airflow.providers.amazon.aws.operators.s3 import (
+ S3CreateBucketOperator,
+ S3CreateObjectOperator,
+ S3DeleteBucketOperator,
+)
+from airflow.utils.trigger_rule import TriggerRule
from tests.system.providers.amazon.aws.utils import SystemTestContextBuilder
-DAG_ID = "example_databrew"
+DAG_ID = "example_glue_databrew"
+
+# Externally fetched variables:
+ROLE_ARN_KEY = "ROLE_ARN"
+
+sys_test_context_task =
SystemTestContextBuilder().add_variable(ROLE_ARN_KEY).build()
+
+EXAMPLE_JSON = "{}"
+
+
+@task
+def create_dataset(dataset_name: str, bucket_name: str, object_key: str):
+ client = boto3.client("databrew")
+ client.create_dataset(
+ Name=dataset_name,
+ Format="JSON",
+ FormatOptions={
+ "Json": {"MultiLine": False},
+ },
+ Input={
+ "S3InputDefinition": {
+ "Bucket": bucket_name,
+ "Key": object_key,
+ },
+ },
+ )
+
+
+@task
+def create_job(
+ dataset_name: str, job_name: str, bucket_output_name: str,
object_output_key: str, role_arn: str
+):
+ client = boto3.client("databrew")
+ client.create_profile_job(
+ DatasetName=dataset_name,
+ Name=job_name,
+ LogSubscription="ENABLE",
+ OutputLocation={
+ "Bucket": bucket_output_name,
+ "Key": object_output_key,
+ },
+ RoleArn=role_arn,
+ )
-sys_test_context_task = SystemTestContextBuilder().build()
+
+@task(trigger_rule=TriggerRule.ALL_DONE)
+def delete_dataset(dataset_name: str):
+ client = boto3.client("databrew")
+ client.delete_dataset(Name=dataset_name)
+
+
+@task(trigger_rule=TriggerRule.ALL_DONE)
+def delete_job(job_name: str):
+ client = boto3.client("databrew")
+ client.delete_job(Name=job_name)
with DAG(DAG_ID, schedule="@once", start_date=pendulum.datetime(2023, 1, 1,
tz="UTC"), catchup=False) as dag:
test_context = sys_test_context_task()
env_id = test_context["ENV_ID"]
+ role_arn = test_context[ROLE_ARN_KEY]
+ bucket_name = f"{env_id}-bucket-databrew"
+ bucket_output_name = f"{env_id}-bucket-output-databrew"
Review Comment:
Here and elsewhere, maybe consider "output_bucket" instead of
"bucket_output" but I'll leave that up to you.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]