This is an automated email from the ASF dual-hosted git repository.
dabla pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 93aed9ad738 Fix templated page_size staying a string in mapped
SqlToS3Operator tasks (#77153)
93aed9ad738 is described below
commit 93aed9ad7389bffafb075c3c22636254413febc6
Author: Pavan Nandi <[email protected]>
AuthorDate: Tue Nov 17 02:09:33 2026 +0530
Fix templated page_size staying a string in mapped SqlToS3Operator tasks
(#77153)
* Fix templated page_size staying a string in mapped SqlToS3Operator tasks
SqlToS3Operator converted the rendered page_size to int in a
render_template_fields override, but a mapped task never calls that
method: MappedOperator renders through _do_render_template_fields on the
unmapped task. page_size stayed a string, so a mapped task uploaded the
first page and then failed comparing it with the page length.
Convert page_size in _do_render_template_fields instead, which both
plain and mapped tasks go through.
* Test the negative page_size check for mapped SqlToS3Operator tasks
---
.../providers/amazon/aws/transfers/sql_to_s3.py | 13 +++++--
.../unit/amazon/aws/transfers/test_sql_to_s3.py | 45 +++++++++++++++++++++-
2 files changed, 53 insertions(+), 5 deletions(-)
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
index 90ab13c0516..9abe506ad35 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
@@ -269,14 +269,19 @@ class SqlToS3Operator(BaseOperator):
"""Format the paginated SQL statement using the current format."""
return self.paginated_sql_statement_clause.format(self.query,
self.page_size, offset)
- def render_template_fields(
+ def _do_render_template_fields(
self,
+ parent: Any,
+ template_fields: Iterable[str],
context: Context,
- jinja_env: jinja2.Environment | None = None,
+ jinja_env: jinja2.Environment,
+ seen_oids: set[int],
) -> None:
- super().render_template_fields(context=context, jinja_env=jinja_env)
+ super()._do_render_template_fields(parent, template_fields, context,
jinja_env, seen_oids)
+ # Hooked here rather than in render_template_fields because a mapped
task never calls
+ # that one -- MappedOperator renders through
_do_render_template_fields on the unmapped task.
# A templated page_size renders as a string; the paging decision needs
the number.
- if self.page_size is not None:
+ if parent is self and self.page_size is not None:
self.page_size = int(self.page_size)
if self.page_size < 0:
raise ValueError(f"SqlToS3Operator argument page_size can not
be negative: {self.page_size}")
diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
b/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
index 81890fdcabc..5e356d7c93c 100644
--- a/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
@@ -31,9 +31,15 @@ from packaging.version import Version
from airflow.exceptions import AirflowProviderDeprecationWarning
from airflow.models import Connection
from airflow.providers.amazon.aws.transfers.sql_to_s3 import SqlToS3Operator
-from airflow.providers.common.compat.sdk import AirflowException,
AirflowOptionalProviderFeatureException
+from airflow.providers.common.compat.sdk import (
+ DAG,
+ AirflowException,
+ AirflowOptionalProviderFeatureException,
+)
from airflow.providers.common.sql.hooks.sql import DbApiHook
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
+
PANDAS_3_PLUS = Version(pd.__version__).major >= 3
PARQUET_ENGINES = ("pyarrow", "fastparquet")
@@ -688,6 +694,43 @@ class TestSqlToS3Operator:
with pytest.raises(ValueError, match="page_size can not be negative"):
op.render_template_fields({})
+ @staticmethod
+ def _mapped_runtime_ti(create_runtime_ti, page_size):
+ with DAG(dag_id="dag", schedule=None):
+ mapped = SqlToS3Operator.partial(
+ task_id="task_id",
+ query="query",
+ s3_bucket="bucket",
+ sql_conn_id="mysql_conn_id",
+ page_size=page_size,
+ ).expand(s3_key=["key"])
+ return create_runtime_ti(task=mapped, map_index=0)
+
+ @pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="create_runtime_ti
needs the Task SDK")
+ @pytest.mark.parametrize(
+ ("page_size", "expected"),
+ [
+ pytest.param("{{ 1 + 1 }}", 2, id="templated"),
+ pytest.param("0", 0, id="rendered-zero-means-unpaged"),
+ ],
+ )
+ def test_render_templates_converts_page_size_for_mapped_task(
+ self, create_runtime_ti, page_size, expected
+ ):
+ """A mapped task never calls render_template_fields, so the conversion
must not rely on it."""
+ ti = self._mapped_runtime_ti(create_runtime_ti, page_size)
+
+ ti.render_templates(context=ti.get_template_context())
+
+ assert ti.task.page_size == expected
+
+ @pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="create_runtime_ti
needs the Task SDK")
+ def
test_render_templates_rejects_a_negative_page_size_for_mapped_task(self,
create_runtime_ti):
+ ti = self._mapped_runtime_ti(create_runtime_ti, "-1")
+
+ with pytest.raises(ValueError, match="page_size can not be negative"):
+ ti.render_templates(context=ti.get_template_context())
+
@mock.patch("airflow.providers.common.sql.operators.sql.BaseHook.get_connection")
def test_hook_params(self, mock_get_conn):
mock_get_conn.return_value = Connection(conn_id="postgres_test",
conn_type="postgres")