This is an automated email from the ASF dual-hosted git repository.

dabla pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 93aed9ad738 Fix templated page_size staying a string in mapped 
SqlToS3Operator tasks (#77153)
93aed9ad738 is described below

commit 93aed9ad7389bffafb075c3c22636254413febc6
Author: Pavan Nandi <[email protected]>
AuthorDate: Tue Nov 17 02:09:33 2026 +0530

    Fix templated page_size staying a string in mapped SqlToS3Operator tasks 
(#77153)
    
    * Fix templated page_size staying a string in mapped SqlToS3Operator tasks
    
    SqlToS3Operator converted the rendered page_size to int in a
    render_template_fields override, but a mapped task never calls that
    method: MappedOperator renders through _do_render_template_fields on the
    unmapped task. page_size stayed a string, so a mapped task uploaded the
    first page and then failed comparing it with the page length.
    
    Convert page_size in _do_render_template_fields instead, which both
    plain and mapped tasks go through.
    
    * Test the negative page_size check for mapped SqlToS3Operator tasks
---
 .../providers/amazon/aws/transfers/sql_to_s3.py    | 13 +++++--
 .../unit/amazon/aws/transfers/test_sql_to_s3.py    | 45 +++++++++++++++++++++-
 2 files changed, 53 insertions(+), 5 deletions(-)

diff --git 
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py 
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
index 90ab13c0516..9abe506ad35 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py
@@ -269,14 +269,19 @@ class SqlToS3Operator(BaseOperator):
         """Format the paginated SQL statement using the current format."""
         return self.paginated_sql_statement_clause.format(self.query, 
self.page_size, offset)
 
-    def render_template_fields(
+    def _do_render_template_fields(
         self,
+        parent: Any,
+        template_fields: Iterable[str],
         context: Context,
-        jinja_env: jinja2.Environment | None = None,
+        jinja_env: jinja2.Environment,
+        seen_oids: set[int],
     ) -> None:
-        super().render_template_fields(context=context, jinja_env=jinja_env)
+        super()._do_render_template_fields(parent, template_fields, context, 
jinja_env, seen_oids)
+        # Hooked here rather than in render_template_fields because a mapped 
task never calls
+        # that one -- MappedOperator renders through 
_do_render_template_fields on the unmapped task.
         # A templated page_size renders as a string; the paging decision needs 
the number.
-        if self.page_size is not None:
+        if parent is self and self.page_size is not None:
             self.page_size = int(self.page_size)
             if self.page_size < 0:
                 raise ValueError(f"SqlToS3Operator argument page_size can not 
be negative: {self.page_size}")
diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py 
b/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
index 81890fdcabc..5e356d7c93c 100644
--- a/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_sql_to_s3.py
@@ -31,9 +31,15 @@ from packaging.version import Version
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.models import Connection
 from airflow.providers.amazon.aws.transfers.sql_to_s3 import SqlToS3Operator
-from airflow.providers.common.compat.sdk import AirflowException, 
AirflowOptionalProviderFeatureException
+from airflow.providers.common.compat.sdk import (
+    DAG,
+    AirflowException,
+    AirflowOptionalProviderFeatureException,
+)
 from airflow.providers.common.sql.hooks.sql import DbApiHook
 
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
+
 PANDAS_3_PLUS = Version(pd.__version__).major >= 3
 
 PARQUET_ENGINES = ("pyarrow", "fastparquet")
@@ -688,6 +694,43 @@ class TestSqlToS3Operator:
         with pytest.raises(ValueError, match="page_size can not be negative"):
             op.render_template_fields({})
 
+    @staticmethod
+    def _mapped_runtime_ti(create_runtime_ti, page_size):
+        with DAG(dag_id="dag", schedule=None):
+            mapped = SqlToS3Operator.partial(
+                task_id="task_id",
+                query="query",
+                s3_bucket="bucket",
+                sql_conn_id="mysql_conn_id",
+                page_size=page_size,
+            ).expand(s3_key=["key"])
+        return create_runtime_ti(task=mapped, map_index=0)
+
+    @pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="create_runtime_ti 
needs the Task SDK")
+    @pytest.mark.parametrize(
+        ("page_size", "expected"),
+        [
+            pytest.param("{{ 1 + 1 }}", 2, id="templated"),
+            pytest.param("0", 0, id="rendered-zero-means-unpaged"),
+        ],
+    )
+    def test_render_templates_converts_page_size_for_mapped_task(
+        self, create_runtime_ti, page_size, expected
+    ):
+        """A mapped task never calls render_template_fields, so the conversion 
must not rely on it."""
+        ti = self._mapped_runtime_ti(create_runtime_ti, page_size)
+
+        ti.render_templates(context=ti.get_template_context())
+
+        assert ti.task.page_size == expected
+
+    @pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="create_runtime_ti 
needs the Task SDK")
+    def 
test_render_templates_rejects_a_negative_page_size_for_mapped_task(self, 
create_runtime_ti):
+        ti = self._mapped_runtime_ti(create_runtime_ti, "-1")
+
+        with pytest.raises(ValueError, match="page_size can not be negative"):
+            ti.render_templates(context=ti.get_template_context())
+
     
@mock.patch("airflow.providers.common.sql.operators.sql.BaseHook.get_connection")
     def test_hook_params(self, mock_get_conn):
         mock_get_conn.return_value = Connection(conn_id="postgres_test", 
conn_type="postgres")

Reply via email to