This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1ced91f5387 Strip GlueDataQualityOperator ruleset after rendering
(#70331)
1ced91f5387 is described below
commit 1ced91f53878ee1e6ffd9651ad770cd2f72b7a4b
Author: Stefan Wang <[email protected]>
AuthorDate: Fri Jul 24 00:47:25 2026 -0700
Strip GlueDataQualityOperator ruleset after rendering (#70331)
* Strip GlueDataQualityOperator ruleset after rendering
ruleset is a template field, so it is rendered after __init__ runs. The
constructor did self.ruleset = ruleset.strip(), which strips the un-rendered
value. A templated ruleset that renders with surrounding whitespace is
therefore
never stripped, and validate_inputs() rejects it because the rendered value
no
longer ends with ']':
AttributeError: RuleSet must starts with Rules = [ and ends with ]
Store ruleset verbatim in the constructor and strip the rendered value at
the
start of execute(), before validation.
related: #70296
---
.../airflow/providers/amazon/aws/operators/glue.py | 4 +++-
.../tests/unit/amazon/aws/operators/test_glue.py | 24 ++++++++++++++++++++++
.../ci/prek/validate_operators_init_exemptions.txt | 1 -
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
index 3d46aecb32e..30077380eb6 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
@@ -433,7 +433,7 @@ class
GlueDataQualityOperator(AwsBaseOperator[GlueDataQualityHook]):
):
super().__init__(**kwargs)
self.name = name
- self.ruleset = ruleset.strip()
+ self.ruleset = ruleset
self.description = description
self.update_rule_set = update_rule_set
self.data_quality_ruleset_kwargs = data_quality_ruleset_kwargs or {}
@@ -449,6 +449,8 @@ class
GlueDataQualityOperator(AwsBaseOperator[GlueDataQualityHook]):
raise AttributeError("Target table must have DatabaseName and
TableName")
def execute(self, context: Context):
+ # ruleset is a template field; strip the rendered value here, not in
__init__.
+ self.ruleset = self.ruleset.strip()
self.validate_inputs()
config = {
diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py
b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py
index 230193feeb4..99d22e3bb5e 100644
--- a/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py
+++ b/providers/amazon/tests/unit/amazon/aws/operators/test_glue.py
@@ -17,6 +17,7 @@
from __future__ import annotations
from collections.abc import Generator
+from datetime import datetime
from typing import TYPE_CHECKING
from unittest import mock
@@ -25,6 +26,7 @@ import pytest
from boto3 import client
from moto import mock_aws
+from airflow.models.dag import DAG
from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook,
GlueJobHook
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink
@@ -813,6 +815,28 @@ class TestGlueDataQualityOperator:
Ruleset=self.RULE_SET,
)
+ @mock.patch.object(GlueDataQualityHook, "conn")
+ def test_execute_strips_rendered_ruleset(self,
glue_data_quality_mock_conn):
+ # ruleset is a template field; execute strips the rendered value
(rendering can add whitespace).
+ with DAG("glue_dq_strip", schedule=None, start_date=datetime(2020, 1,
1)) as dag:
+ self.operator = GlueDataQualityOperator(
+ task_id="create_data_quality_ruleset",
+ name=self.RULE_SET_NAME,
+ ruleset="{{ params.rules }}",
+ dag=dag,
+ )
+ self.operator.defer = mock.MagicMock()
+ self.operator.render_template_fields({"params": {"rules": f"
{self.RULE_SET} "}})
+ assert self.operator.ruleset == f" {self.RULE_SET} "
+
+ self.operator.execute({})
+
+
glue_data_quality_mock_conn.create_data_quality_ruleset.assert_called_once_with(
+ Description="AWS Glue Data Quality Rule Set With Airflow",
+ Name=self.RULE_SET_NAME,
+ Ruleset=self.RULE_SET,
+ )
+
@mock.patch.object(GlueDataQualityHook, "conn")
def test_execute_create_rule_should_fail_if_rule_already_exists(self,
glue_data_quality_mock_conn):
self.operator = GlueDataQualityOperator(
diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt
b/scripts/ci/prek/validate_operators_init_exemptions.txt
index 669597d0a3d..3203e08b306 100644
--- a/scripts/ci/prek/validate_operators_init_exemptions.txt
+++ b/scripts/ci/prek/validate_operators_init_exemptions.txt
@@ -11,7 +11,6 @@
providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsModifyTas
providers/amazon/src/airflow/providers/amazon/aws/operators/dms.py::DmsStartReplicationOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/ecs.py::EcsRunTaskOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py::EmrAddStepsOperator
-providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py::GlueDataQualityOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/s3.py::S3DeleteObjectsOperator
