This is an automated email from the ASF dual-hosted git repository.

eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 463e28b9ce add `env_vars` into `_BasePythonVirtualenvOperator` (#40942)
463e28b9ce is described below

commit 463e28b9ce6b58a64795c96ae7b94aba37770c93
Author: Youngha, Park <[email protected]>
AuthorDate: Fri Aug 9 00:09:13 2024 +0900

    add `env_vars` into `_BasePythonVirtualenvOperator` (#40942)
    
    * add env_vars into _BasePythonVirtualenvOperator
    
    * fix BranchPythonVirtualenvOperator
    
    * update feedback
---
 airflow/decorators/__init__.pyi   | 16 +++++++
 airflow/operators/python.py       | 31 ++++++++++++-
 airflow/utils/process_utils.py    |  7 ++-
 tests/operators/test_python.py    | 93 +++++++++++++++++++++++++++++++++++++++
 tests/utils/test_process_utils.py |  4 ++
 5 files changed, 148 insertions(+), 3 deletions(-)

diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi
index 9b4fbccc2e..faf77e8240 100644
--- a/airflow/decorators/__init__.pyi
+++ b/airflow/decorators/__init__.pyi
@@ -122,6 +122,8 @@ class TaskDecoratorCollection:
         index_urls: None | Collection[str] | str = None,
         venv_cache_path: None | str = None,
         show_return_value_in_logs: bool = True,
+        env_vars: dict[str, str] | None = None,
+        inherit_env: bool = True,
         use_dill: bool = False,
         **kwargs,
     ) -> TaskDecorator:
@@ -165,6 +167,12 @@ class TaskDecoratorCollection:
             logs. Defaults to True, which allows return value log output.
             It can be set to False to prevent log output of return value when 
you return huge data
             such as transmission a large amount of XCom to TaskAPI.
+        :param env_vars: A dictionary containing additional environment 
variables to set for the virtual
+            environment when it is executed.
+        :param inherit_env: Whether to inherit the current environment 
variables when executing the virtual
+            environment. If set to ``True``, the virtual environment will 
inherit the environment variables
+            of the parent process (``os.environ``). If set to ``False``, the 
virtual environment will be
+            executed with a clean environment.
         :param use_dill: Deprecated, use ``serializer`` instead. Whether to 
use dill to serialize
             the args and result (pickle is default). This allows more complex 
types
             but requires you to include dill in your requirements.
@@ -181,6 +189,8 @@ class TaskDecoratorCollection:
         serializer: Literal["pickle", "cloudpickle", "dill"] | None = None,
         templates_dict: Mapping[str, Any] | None = None,
         show_return_value_in_logs: bool = True,
+        env_vars: dict[str, str] | None = None,
+        inherit_env: bool = True,
         use_dill: bool = False,
         **kwargs,
     ) -> TaskDecorator:
@@ -206,6 +216,12 @@ class TaskDecoratorCollection:
             logs. Defaults to True, which allows return value log output.
             It can be set to False to prevent log output of return value when 
you return huge data
             such as transmission a large amount of XCom to TaskAPI.
+        :param env_vars: A dictionary containing additional environment 
variables to set for the virtual
+            environment when it is executed.
+        :param inherit_env: Whether to inherit the current environment 
variables when executing the virtual
+            environment. If set to ``True``, the virtual environment will 
inherit the environment variables
+            of the parent process (``os.environ``). If set to ``False``, the 
virtual environment will be
+            executed with a clean environment.
         :param use_dill: Deprecated, use ``serializer`` instead. Whether to 
use dill to serialize
             the args and result (pickle is default). This allows more complex 
types
             but requires you to include dill in your requirements.
diff --git a/airflow/operators/python.py b/airflow/operators/python.py
index 46dbff9f01..fdfe575fb9 100644
--- a/airflow/operators/python.py
+++ b/airflow/operators/python.py
@@ -439,6 +439,8 @@ class _BasePythonVirtualenvOperator(PythonOperator, 
metaclass=ABCMeta):
         templates_exts: list[str] | None = None,
         expect_airflow: bool = True,
         skip_on_exit_code: int | Container[int] | None = None,
+        env_vars: dict[str, str] | None = None,
+        inherit_env: bool = True,
         use_dill: bool = False,
         **kwargs,
     ):
@@ -490,6 +492,8 @@ class _BasePythonVirtualenvOperator(PythonOperator, 
metaclass=ABCMeta):
             if skip_on_exit_code is not None
             else []
         )
+        self.env_vars = env_vars
+        self.inherit_env = inherit_env
 
     @abstractmethod
     def _iter_serializable_context_keys(self):
@@ -558,6 +562,10 @@ class _BasePythonVirtualenvOperator(PythonOperator, 
metaclass=ABCMeta):
                 
render_template_as_native_obj=self.dag.render_template_as_native_obj,
             )
 
+            env_vars = dict(os.environ) if self.inherit_env else {}
+            if self.env_vars:
+                env_vars.update(self.env_vars)
+
             try:
                 execute_in_subprocess(
                     cmd=[
@@ -567,7 +575,8 @@ class _BasePythonVirtualenvOperator(PythonOperator, 
metaclass=ABCMeta):
                         os.fspath(output_path),
                         os.fspath(string_args_path),
                         os.fspath(termination_log_path),
-                    ]
+                    ],
+                    env=env_vars,
                 )
             except subprocess.CalledProcessError as e:
                 if e.returncode in self.skip_on_exit_code:
@@ -648,6 +657,12 @@ class 
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
         virtual environment will be cached, creates a sub-folder venv-{hash} 
whereas hash will be replaced
         with a checksum of requirements. If not provided the virtual 
environment will be created and deleted
         in a temp folder for every execution.
+    :param env_vars: A dictionary containing additional environment variables 
to set for the virtual
+        environment when it is executed.
+    :param inherit_env: Whether to inherit the current environment variables 
when executing the virtual
+        environment. If set to ``True``, the virtual environment will inherit 
the environment variables
+        of the parent process (``os.environ``). If set to ``False``, the 
virtual environment will be
+        executed with a clean environment.
     :param use_dill: Deprecated, use ``serializer`` instead. Whether to use 
dill to serialize
         the args and result (pickle is default). This allows more complex types
         but requires you to include dill in your requirements.
@@ -676,6 +691,8 @@ class 
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
         skip_on_exit_code: int | Container[int] | None = None,
         index_urls: None | Collection[str] | str = None,
         venv_cache_path: None | os.PathLike[str] = None,
+        env_vars: dict[str, str] | None = None,
+        inherit_env: bool = True,
         use_dill: bool = False,
         **kwargs,
     ):
@@ -724,6 +741,8 @@ class 
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
             templates_exts=templates_exts,
             expect_airflow=expect_airflow,
             skip_on_exit_code=skip_on_exit_code,
+            env_vars=env_vars,
+            inherit_env=inherit_env,
             use_dill=use_dill,
             **kwargs,
         )
@@ -934,6 +953,12 @@ class 
ExternalPythonOperator(_BasePythonVirtualenvOperator):
     :param skip_on_exit_code: If python_callable exits with this exit code, 
leave the task
         in ``skipped`` state (default: None). If set to ``None``, any non-zero
         exit code will be treated as a failure.
+    :param env_vars: A dictionary containing additional environment variables 
to set for the virtual
+        environment when it is executed.
+    :param inherit_env: Whether to inherit the current environment variables 
when executing the virtual
+        environment. If set to ``True``, the virtual environment will inherit 
the environment variables
+        of the parent process (``os.environ``). If set to ``False``, the 
virtual environment will be
+        executed with a clean environment.
     :param use_dill: Deprecated, use ``serializer`` instead. Whether to use 
dill to serialize
         the args and result (pickle is default). This allows more complex types
         but requires you to include dill in your requirements.
@@ -955,6 +980,8 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
         expect_airflow: bool = True,
         expect_pendulum: bool = False,
         skip_on_exit_code: int | Container[int] | None = None,
+        env_vars: dict[str, str] | None = None,
+        inherit_env: bool = True,
         use_dill: bool = False,
         **kwargs,
     ):
@@ -972,6 +999,8 @@ class ExternalPythonOperator(_BasePythonVirtualenvOperator):
             templates_exts=templates_exts,
             expect_airflow=expect_airflow,
             skip_on_exit_code=skip_on_exit_code,
+            env_vars=env_vars,
+            inherit_env=inherit_env,
             use_dill=use_dill,
             **kwargs,
         )
diff --git a/airflow/utils/process_utils.py b/airflow/utils/process_utils.py
index e8b3e8840d..7120bd5df8 100644
--- a/airflow/utils/process_utils.py
+++ b/airflow/utils/process_utils.py
@@ -162,14 +162,17 @@ def reap_process_group(
     return returncodes
 
 
-def execute_in_subprocess(cmd: list[str], cwd: str | None = None) -> None:
+def execute_in_subprocess(cmd: list[str], cwd: str | None = None, env: dict | 
None = None) -> None:
     """
     Execute a process and stream output to logger.
 
     :param cmd: command and arguments to run
     :param cwd: Current working directory passed to the Popen constructor
+    :param env: Additional environment variables to set for the subprocess. If 
None,
+        the subprocess will inherit the current environment variables of the 
parent process
+        (``os.environ``)
     """
-    execute_in_subprocess_with_kwargs(cmd, cwd=cwd)
+    execute_in_subprocess_with_kwargs(cmd, cwd=cwd, env=env)
 
 
 def execute_in_subprocess_with_kwargs(cmd: list[str], **kwargs) -> None:
diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py
index 893a215d9d..993d70cad3 100644
--- a/tests/operators/test_python.py
+++ b/tests/operators/test_python.py
@@ -963,6 +963,48 @@ class BaseTestPythonVirtualenvOperator(BasePythonTest):
             self.run_as_task(f, op_args=[42], serializer=serializer)
         assert f"Unable to import `{serializer}` module." in caplog.text
 
+    def test_environment_variables(self):
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        task = self.run_as_task(f, env_vars={"MY_ENV_VAR": "ABCDE"})
+        assert task.execute_callable() == "ABCDE"
+
+    def test_environment_variables_with_inherit_env_true(self, monkeypatch):
+        monkeypatch.setenv("MY_ENV_VAR", "QWERT")
+
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        task = self.run_as_task(f, inherit_env=True)
+        assert task.execute_callable() == "QWERT"
+
+    def test_environment_variables_with_inherit_env_false(self, monkeypatch):
+        monkeypatch.setenv("MY_ENV_VAR", "TYUIO")
+
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        with pytest.raises(AirflowException):
+            self.run_as_task(f, inherit_env=False)
+
+    def test_environment_variables_overriding(self, monkeypatch):
+        monkeypatch.setenv("MY_ENV_VAR", "ABCDE")
+
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        task = self.run_as_task(f, env_vars={"MY_ENV_VAR": "EFGHI"}, 
inherit_env=True)
+        assert task.execute_callable() == "EFGHI"
+
 
 venv_cache_path = tempfile.mkdtemp(prefix="venv_cache_path")
 
@@ -1489,6 +1531,57 @@ class 
BaseTestBranchPythonVirtualenvOperator(BaseTestPythonVirtualenvOperator):
         with pytest.raises(AirflowException, match="Invalid tasks found:"):
             self.run_as_task(f, templates_dict={"ds": "{{ ds }}"})
 
+    def test_environment_variables(self):
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        with pytest.raises(
+            AirflowException,
+            match=r"'branch_task_ids' must contain only valid task_ids. 
Invalid tasks found: {'ABCDE'}",
+        ):
+            self.run_as_task(f, env_vars={"MY_ENV_VAR": "ABCDE"})
+
+    def test_environment_variables_with_inherit_env_true(self, monkeypatch):
+        monkeypatch.setenv("MY_ENV_VAR", "QWERT")
+
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        with pytest.raises(
+            AirflowException,
+            match=r"'branch_task_ids' must contain only valid task_ids. 
Invalid tasks found: {'QWERT'}",
+        ):
+            self.run_as_task(f, inherit_env=True)
+
+    def test_environment_variables_with_inherit_env_false(self, monkeypatch):
+        monkeypatch.setenv("MY_ENV_VAR", "TYUIO")
+
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        with pytest.raises(AirflowException):
+            self.run_as_task(f, inherit_env=False)
+
+    def test_environment_variables_overriding(self, monkeypatch):
+        monkeypatch.setenv("MY_ENV_VAR", "ABCDE")
+
+        def f():
+            import os
+
+            return os.environ["MY_ENV_VAR"]
+
+        with pytest.raises(
+            AirflowException,
+            match=r"'branch_task_ids' must contain only valid task_ids. 
Invalid tasks found: {'EFGHI'}",
+        ):
+            self.run_as_task(f, env_vars={"MY_ENV_VAR": "EFGHI"}, 
inherit_env=True)
+
     def test_with_no_caching(self):
         """
         Most of venv tests use caching to speed up the tests. This test 
ensures that
diff --git a/tests/utils/test_process_utils.py 
b/tests/utils/test_process_utils.py
index 54bdc5bf9f..bf9ecf4352 100644
--- a/tests/utils/test_process_utils.py
+++ b/tests/utils/test_process_utils.py
@@ -120,6 +120,10 @@ class TestExecuteInSubProcess:
             "KITTY",
         ] == msgs
 
+    def test_using_env_works(self, caplog):
+        execute_in_subprocess(["bash", "-c", 'echo "My value is ${VALUE}"'], 
env=dict(VALUE="1"))
+        assert "My value is 1" in caplog.text
+
     def test_should_raise_exception(self):
         with pytest.raises(CalledProcessError):
             process_utils.execute_in_subprocess(["bash", "-c", "exit 1"])

Reply via email to