This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 46c0f85ba6 Add "literal" wrapper to disable field templating (#35017)
46c0f85ba6 is described below
commit 46c0f85ba6dd654501fc429ddd831461ebfefd3c
Author: Michał Sośnicki <[email protected]>
AuthorDate: Fri Nov 17 09:58:49 2023 +0100
Add "literal" wrapper to disable field templating (#35017)
Co-authored-by: Tzu-ping Chung <[email protected]>
---
airflow/template/templater.py | 19 +++++++++++++++
airflow/utils/template.py | 32 +++++++++++++++++++++++++
docs/apache-airflow/core-concepts/operators.rst | 29 +++++++++++++++++-----
tests/models/test_baseoperator.py | 4 ++++
tests/template/test_templater.py | 22 ++++++++++++++++-
5 files changed, 99 insertions(+), 7 deletions(-)
diff --git a/airflow/template/templater.py b/airflow/template/templater.py
index 9cb6a312ad..f4596f8da1 100644
--- a/airflow/template/templater.py
+++ b/airflow/template/templater.py
@@ -17,6 +17,7 @@
from __future__ import annotations
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Collection, Iterable, Sequence
from airflow.utils.helpers import render_template_as_native,
render_template_to_string
@@ -29,9 +30,27 @@ if TYPE_CHECKING:
from sqlalchemy.orm import Session
from airflow import DAG
+ from airflow.models.operator import Operator
from airflow.utils.context import Context
+@dataclass(frozen=True)
+class LiteralValue(ResolveMixin):
+ """
+ A wrapper for a value that should be rendered as-is, without applying
jinja templating to its contents.
+
+ :param value: The value to be rendered without templating
+ """
+
+ value: Any
+
+ def iter_references(self) -> Iterable[tuple[Operator, str]]:
+ return ()
+
+ def resolve(self, context: Context) -> Any:
+ return self.value
+
+
class Templater(LoggingMixin):
"""
This renders the template fields of object.
diff --git a/airflow/utils/template.py b/airflow/utils/template.py
new file mode 100644
index 0000000000..ae6042a7e9
--- /dev/null
+++ b/airflow/utils/template.py
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import Any
+
+from airflow.template.templater import LiteralValue
+
+
+def literal(value: Any) -> LiteralValue:
+ """
+ Wrap a value to ensure it is rendered as-is without applying Jinja
templating to its contents.
+
+ Designed for use in an operator's template field.
+
+ :param value: The value to be rendered without templating
+ """
+ return LiteralValue(value)
diff --git a/docs/apache-airflow/core-concepts/operators.rst
b/docs/apache-airflow/core-concepts/operators.rst
index ddbcf7da7a..c7193789bc 100644
--- a/docs/apache-airflow/core-concepts/operators.rst
+++ b/docs/apache-airflow/core-concepts/operators.rst
@@ -158,37 +158,54 @@ See the `Jinja documentation
<https://jinja.palletsprojects.com/en/2.11.x/api/#j
Some operators will also consider strings ending in specific suffixes (defined
in ``template_ext``) to be references to files when rendering fields. This can
be useful for loading scripts or queries directly from files rather than
including them into DAG code.
-For example, consider a BashOperator which runs a multi-line bash script, this
will load the file at ``script.sh`` and use its contents as the value for
``bash_callable``:
+For example, consider a BashOperator which runs a multi-line bash script, this
will load the file at ``script.sh`` and use its contents as the value for
``bash_command``:
.. code-block:: python
run_script = BashOperator(
task_id="run_script",
- bash_callable="script.sh",
+ bash_command="script.sh",
)
By default, paths provided in this way should be provided relative to the
DAG's folder (as this is the default Jinja template search path), but
additional paths can be added by setting the ``template_searchpath`` arg on the
DAG.
-In some cases you may want to disable template rendering on specific fields or
prevent airflow from trying to read template files for a given suffix. Consider
the following task:
+In some cases, you may want to exclude a string from templating and use it
directly. Consider the following task:
.. code-block:: python
print_script = BashOperator(
task_id="print_script",
- bash_callable="cat script.sh",
+ bash_command="cat script.sh",
)
+This will fail with ``TemplateNotFound: cat script.sh`` since Airflow would
treat the string as a path to a file, not a command.
+We can prevent airflow from treating this value as a reference to a file by
wrapping it in :func:`~airflow.util.template.literal`.
+This approach disables the rendering of both macros and files and can be
applied to selected nested fields while retaining the default templating rules
for the remainder of the content.
-This will fail with ``TemplateNotFound: cat script.sh``, but we can prevent
airflow from treating this value as a reference to a file by overriding
``template_ext``:
+.. code-block:: python
+
+ from airflow.utils.template import literal
+
+
+ fixed_print_script = BashOperator(
+ task_id="fixed_print_script",
+ bash_command=literal("cat script.sh"),
+ )
+
+.. versionadded:: 2.8
+ :func:`~airflow.util.template.literal` was added.
+
+Alternatively, if you want to prevent Airflow from treating a value as a
reference to a file, you can override ``template_ext``:
.. code-block:: python
fixed_print_script = BashOperator(
task_id="fixed_print_script",
- bash_callable="cat script.sh",
+ bash_command="cat script.sh",
)
fixed_print_script.template_ext = ()
+
.. _concepts:templating-native-objects:
Rendering Fields as Native Python Objects
diff --git a/tests/models/test_baseoperator.py
b/tests/models/test_baseoperator.py
index 793a31b465..fb46fd39c7 100644
--- a/tests/models/test_baseoperator.py
+++ b/tests/models/test_baseoperator.py
@@ -43,6 +43,7 @@ from airflow.models.dagrun import DagRun
from airflow.models.taskinstance import TaskInstance
from airflow.utils.edgemodifier import Label
from airflow.utils.task_group import TaskGroup
+from airflow.utils.template import literal
from airflow.utils.trigger_rule import TriggerRule
from airflow.utils.types import DagRunType
from airflow.utils.weight_rule import WeightRule
@@ -277,6 +278,9 @@ class TestBaseOperator:
),
# By default, Jinja2 drops one (single) trailing newline
("{{ foo }}\n\n", {"foo": "bar"}, "bar\n"),
+ (literal("{{ foo }}"), {"foo": "bar"}, "{{ foo }}"),
+ (literal(["{{ foo }}_1", "{{ foo }}_2"]), {"foo": "bar"}, ["{{ foo
}}_1", "{{ foo }}_2"]),
+ (literal(("{{ foo }}_1", "{{ foo }}_2")), {"foo": "bar"}, ("{{ foo
}}_1", "{{ foo }}_2")),
],
)
def test_render_template(self, content, context, expected_output):
diff --git a/tests/template/test_templater.py b/tests/template/test_templater.py
index 1c3d848de3..c28e1f0503 100644
--- a/tests/template/test_templater.py
+++ b/tests/template/test_templater.py
@@ -20,7 +20,7 @@ from __future__ import annotations
import jinja2
from airflow.models.dag import DAG
-from airflow.template.templater import Templater
+from airflow.template.templater import LiteralValue, Templater
from airflow.utils.context import Context
@@ -60,3 +60,23 @@ class TestTemplater:
templater.template_ext = [".txt"]
rendered_content = templater.render_template(templater.message,
context)
assert rendered_content == "Hello world"
+
+ def test_not_render_literal_value(self):
+ templater = Templater()
+ templater.template_ext = []
+ context = Context()
+ content = LiteralValue("Hello {{ name }}")
+
+ rendered_content = templater.render_template(content, context)
+
+ assert rendered_content == "Hello {{ name }}"
+
+ def test_not_render_file_literal_value(self):
+ templater = Templater()
+ templater.template_ext = [".txt"]
+ context = Context()
+ content = LiteralValue("template_file.txt")
+
+ rendered_content = templater.render_template(content, context)
+
+ assert rendered_content == "template_file.txt"
