This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 78187ca863 Expose catalog parameter in `AthenaOpeartor` (#35103)
78187ca863 is described below
commit 78187ca86361b96a7c9565da130d9b9b42e93509
Author: Elad Kalif <[email protected]>
AuthorDate: Mon Oct 23 11:44:07 2023 +0300
Expose catalog parameter in `AthenaOpeartor` (#35103)
---
airflow/providers/amazon/aws/operators/athena.py | 6 +++++-
tests/providers/amazon/aws/operators/test_athena.py | 20 +++++++++++++++++++-
2 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/airflow/providers/amazon/aws/operators/athena.py
b/airflow/providers/amazon/aws/operators/athena.py
index e0de3122b0..95340d6f8a 100644
--- a/airflow/providers/amazon/aws/operators/athena.py
+++ b/airflow/providers/amazon/aws/operators/athena.py
@@ -43,6 +43,7 @@ class AthenaOperator(BaseOperator):
:param query: Presto to be run on athena. (templated)
:param database: Database to select. (templated)
+ :param catalog: Catalog to select. (templated)
:param output_location: s3 path to write the query results into.
(templated)
:param aws_conn_id: aws connection to use
:param client_request_token: Unique token created by user to avoid
multiple executions of same query
@@ -57,7 +58,7 @@ class AthenaOperator(BaseOperator):
"""
ui_color = "#44b5e2"
- template_fields: Sequence[str] = ("query", "database", "output_location",
"workgroup")
+ template_fields: Sequence[str] = ("query", "database", "output_location",
"workgroup", "catalog")
template_ext: Sequence[str] = (".sql",)
template_fields_renderers = {"query": "sql"}
@@ -76,6 +77,7 @@ class AthenaOperator(BaseOperator):
max_polling_attempts: int | None = None,
log_query: bool = True,
deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ catalog: str = "AwsDataCatalog",
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
@@ -92,6 +94,7 @@ class AthenaOperator(BaseOperator):
self.query_execution_id: str | None = None
self.log_query: bool = log_query
self.deferrable = deferrable
+ self.catalog: str = catalog
@cached_property
def hook(self) -> AthenaHook:
@@ -101,6 +104,7 @@ class AthenaOperator(BaseOperator):
def execute(self, context: Context) -> str | None:
"""Run Presto Query on Athena."""
self.query_execution_context["Database"] = self.database
+ self.query_execution_context["Catalog"] = self.catalog
self.result_configuration["OutputLocation"] = self.output_location
self.query_execution_id = self.hook.run_query(
self.query,
diff --git a/tests/providers/amazon/aws/operators/test_athena.py
b/tests/providers/amazon/aws/operators/test_athena.py
index 3eb6349850..5208c7c683 100644
--- a/tests/providers/amazon/aws/operators/test_athena.py
+++ b/tests/providers/amazon/aws/operators/test_athena.py
@@ -36,12 +36,13 @@ MOCK_DATA = {
"task_id": "test_athena_operator",
"query": "SELECT * FROM TEST_TABLE",
"database": "TEST_DATABASE",
+ "catalog": "AwsDataCatalog",
"outputLocation": "s3://test_s3_bucket/",
"client_request_token": "eac427d0-1c6d-4dfb-96aa-2835d3ac6595",
"workgroup": "primary",
}
-query_context = {"Database": MOCK_DATA["database"]}
+query_context = {"Database": MOCK_DATA["database"], "Catalog":
MOCK_DATA["catalog"]}
result_configuration = {"OutputLocation": MOCK_DATA["outputLocation"]}
@@ -69,10 +70,27 @@ class TestAthenaOperator:
assert self.athena.task_id == MOCK_DATA["task_id"]
assert self.athena.query == MOCK_DATA["query"]
assert self.athena.database == MOCK_DATA["database"]
+ assert self.athena.catalog == MOCK_DATA["catalog"]
assert self.athena.aws_conn_id == "aws_default"
assert self.athena.client_request_token ==
MOCK_DATA["client_request_token"]
assert self.athena.sleep_time == 0
+ @mock.patch.object(AthenaHook, "check_query_status",
side_effect=("SUCCEEDED",))
+ @mock.patch.object(AthenaHook, "run_query", return_value=ATHENA_QUERY_ID)
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_hook_run_override_catalog(self, mock_conn, mock_run_query,
mock_check_query_status):
+ query_context_catalog = {"Database": MOCK_DATA["database"], "Catalog":
"MyCatalog"}
+ self.athena.catalog = "MyCatalog"
+ self.athena.execute({})
+ mock_run_query.assert_called_once_with(
+ MOCK_DATA["query"],
+ query_context_catalog,
+ result_configuration,
+ MOCK_DATA["client_request_token"],
+ MOCK_DATA["workgroup"],
+ )
+ assert mock_check_query_status.call_count == 1
+
@mock.patch.object(AthenaHook, "check_query_status",
side_effect=("SUCCEEDED",))
@mock.patch.object(AthenaHook, "run_query", return_value=ATHENA_QUERY_ID)
@mock.patch.object(AthenaHook, "get_conn")