This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2b5d63c4020 Support Athena managed query results in AthenaHook (#77067)
2b5d63c4020 is described below
commit 2b5d63c4020df0267329e0a563b97f64a75e8a21
Author: Sean Ghaeli <[email protected]>
AuthorDate: Tue Nov 17 14:31:42 2026 -0700
Support Athena managed query results in AthenaHook (#77067)
A workgroup with managed query results keeps the results in Athena, so its
query executions have no ResultConfiguration and get_output_location raised
KeyError. It now returns None for them. run_query also stops sending an
empty ResultConfiguration, so one is only sent when an output location or
encryption is set.
---
.../airflow/providers/amazon/aws/hooks/athena.py | 18 +++++---
.../providers/amazon/aws/operators/athena.py | 10 +++--
.../tests/unit/amazon/aws/hooks/test_athena.py | 51 ++++++++++++++++++++++
3 files changed, 70 insertions(+), 9 deletions(-)
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/athena.py
b/providers/amazon/src/airflow/providers/amazon/aws/hooks/athena.py
index 7d8ab21fa82..5f5ed86b8d8 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/athena.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/athena.py
@@ -110,7 +110,7 @@ class AthenaHook(AwsBaseHook):
self,
query: str,
query_context: dict[str, str],
- result_configuration: dict[str, Any],
+ result_configuration: dict[str, Any] | None = None,
client_request_token: str | None = None,
workgroup: str = "primary",
) -> str:
@@ -123,7 +123,8 @@ class AthenaHook(AwsBaseHook):
:param query: Trino/Presto query to run.
:param query_context: Context in which query need to be run.
:param result_configuration: Dict with path to store results in and
- config related to encryption.
+ config related to encryption. Leave it unset for a workgroup with
+ managed query results, which keeps the results in Athena.
:param client_request_token: Unique token created by user to avoid
multiple executions of same query.
:param workgroup: Athena workgroup name, when not specified, will be
``'primary'``.
@@ -132,9 +133,10 @@ class AthenaHook(AwsBaseHook):
params = {
"QueryString": query,
"QueryExecutionContext": query_context,
- "ResultConfiguration": result_configuration,
"WorkGroup": workgroup,
}
+ if result_configuration:
+ params["ResultConfiguration"] = result_configuration
if client_request_token:
params["ClientRequestToken"] = client_request_token
if self.log_query:
@@ -327,10 +329,13 @@ class AthenaHook(AwsBaseHook):
)
return self.check_query_status(query_execution_id)
- def get_output_location(self, query_execution_id: str) -> str:
+ def get_output_location(self, query_execution_id: str) -> str | None:
"""
Get the output location of the query results in S3 URI format.
+ Returns None when the workgroup has managed query results enabled,
since Athena
+ stores those results itself instead of in S3.
+
.. seealso::
- :external+boto3:py:meth:`Athena.Client.get_query_execution`
@@ -343,7 +348,10 @@ class AthenaHook(AwsBaseHook):
raise ValueError(f"Unable to get query information for execution
id: {query_execution_id}")
try:
- return
response["QueryExecution"]["ResultConfiguration"]["OutputLocation"]
+ query_execution = response["QueryExecution"]
+ if query_execution.get("ManagedQueryResultsConfiguration",
{}).get("Enabled"):
+ return None
+ return query_execution["ResultConfiguration"]["OutputLocation"]
except KeyError:
self.log.error("Error retrieving OutputLocation. Query execution
id: %s", query_execution_id)
raise
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/operators/athena.py
b/providers/amazon/src/airflow/providers/amazon/aws/operators/athena.py
index 592dc78ece1..58f606aaccb 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/athena.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/athena.py
@@ -54,10 +54,12 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
the ``query_execution_context`` will be used instead.
:param catalog: Catalog to select. (templated)
:param output_location: s3 path to write the query results into.
(templated)
- To run the query, you must specify the query results location using
one of the ways:
- either for individual queries using either this setting (client-side),
- or in the workgroup, using WorkGroupConfiguration.
- If none of them is set, Athena issues an error that no output location
is provided
+ The query results location comes from one of three places: this setting
+ (client-side), the workgroup, using WorkGroupConfiguration, or Athena
itself
+ for a workgroup with managed query results. Leave this setting unset
for such
+ a workgroup, since Athena rejects a query that also sets an output
location.
+ If none of the three applies, Athena issues an error that no output
location
+ is provided.
:param client_request_token: Unique token created by user to avoid
multiple executions of same query
:param workgroup: Athena workgroup in which query will be run. (templated)
:param query_execution_context: Context in which query need to be run
diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_athena.py
b/providers/amazon/tests/unit/amazon/aws/hooks/test_athena.py
index 4e09bbec92a..4043ffae318 100644
--- a/providers/amazon/tests/unit/amazon/aws/hooks/test_athena.py
+++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_athena.py
@@ -116,6 +116,27 @@ class TestAthenaHook:
mock_conn.return_value.start_query_execution.assert_called_with(**expected_call_params)
assert result == MOCK_DATA["query_execution_id"]
+ @pytest.mark.parametrize(
+ "extra_kwargs",
+ [
+ pytest.param({}, id="omitted"),
+ pytest.param({"result_configuration": {}}, id="empty-dict"),
+ ],
+ )
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_hook_run_query_omits_unset_result_configuration(self, mock_conn,
extra_kwargs):
+ mock_conn.return_value.start_query_execution.return_value =
MOCK_QUERY_EXECUTION
+ self.athena.run_query(
+ query=MOCK_DATA["query"],
+ query_context=mock_query_context,
+ **extra_kwargs,
+ )
+ mock_conn.return_value.start_query_execution.assert_called_with(
+ QueryString=MOCK_DATA["query"],
+ QueryExecutionContext=mock_query_context,
+ WorkGroup=MOCK_DATA["workgroup"],
+ )
+
@mock.patch.object(AthenaHook, "log")
@mock.patch.object(AthenaHook, "get_conn")
def test_hook_run_query_log_query(self, mock_conn, log):
@@ -246,6 +267,36 @@ class TestAthenaHook:
result =
self.athena.get_output_location(query_execution_id=MOCK_DATA["query_execution_id"])
assert result == "s3://test_bucket/test.csv"
+ @mock.patch.object(AthenaHook, "get_conn")
+ def test_hook_get_output_location_managed_query_results(self, mock_conn):
+ mock_conn.return_value.get_query_execution.return_value = {
+ "QueryExecution": {
+ "QueryExecutionId": MOCK_DATA["query_execution_id"],
+ "ManagedQueryResultsConfiguration": {"Enabled": True},
+ }
+ }
+ assert
self.athena.get_output_location(query_execution_id=MOCK_DATA["query_execution_id"])
is None
+
+ @pytest.mark.parametrize(
+ "query_execution",
+ [
+ pytest.param({"QueryExecutionId":
MOCK_DATA["query_execution_id"]}, id="no-result-config"),
+ pytest.param(
+ {
+ "QueryExecutionId": MOCK_DATA["query_execution_id"],
+ "ManagedQueryResultsConfiguration": {"Enabled": False},
+ "ResultConfiguration": {},
+ },
+ id="managed-results-disabled",
+ ),
+ ],
+ )
+ @mock.patch.object(AthenaHook, "get_conn")
+ def
test_hook_get_output_location_raises_without_managed_query_results(self,
mock_conn, query_execution):
+ mock_conn.return_value.get_query_execution.return_value =
{"QueryExecution": query_execution}
+ with pytest.raises(KeyError):
+
self.athena.get_output_location(query_execution_id=MOCK_DATA["query_execution_id"])
+
@pytest.mark.parametrize(
"query_execution_id", [pytest.param("", id="empty-string"),
pytest.param(None, id="none")]
)