softyoungha commented on code in PR #40775:
URL: https://github.com/apache/airflow/pull/40775#discussion_r1678142109
##########
airflow/providers_manager.py:
##########
@@ -467,7 +468,7 @@ def _init_airflow_core_hooks(self):
connection_type=None,
connection_testable=False,
)
- for cls in [FSHook, PackageIndexHook]:
+ for cls in [FSHook, PackageIndexHook, PythonVirtualenvHook]:
Review Comment:
Given that the virtual environment hook is included in the base hook,
I thought it should be added here.
Is this the right place to add it?
##########
airflow/hooks/python_virtualenv.py:
##########
@@ -0,0 +1,353 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import sys
+from contextlib import contextmanager
+from functools import cached_property
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, Generator
+
+from airflow.exceptions import (
+ AirflowException,
+)
+from airflow.hooks.base import BaseHook
+from airflow.models.variable import Variable
+from airflow.utils import hashlib_wrapper
+from airflow.utils.python_virtualenv import prepare_virtualenv
+from airflow.utils.types import NOTSET, ArgNotSet
+
+if TYPE_CHECKING:
+ from airflow.typing_compat import Literal
+
+log = logging.getLogger(__name__)
+
+
+class PythonVirtualenvHook(BaseHook):
+ """
+ To run a function in a virtualenv, create virtualenv first and destroy
automatically.
+
+ :param venv_conn_id: The ID of Python Virtualenv Connection.
+ If set, the other arguments in the Operator will override the values
from the connection.
+ :param requirements: Either a list of requirement strings, or a (templated)
+ "requirements file" as specified by pip.
+ :param python_version: The Python version to run the virtual environment
with. Note that
+ both 2 and 2.7 are acceptable forms.
+ :param serializer: Which serializer use to serialize the args and result.
It can be one of the following:
+
+ - ``"pickle"``: (default) Use pickle for serialization. Included in
the Python Standard Library.
+ - ``"cloudpickle"``: Use cloudpickle for serialize more complex types,
+ this requires to include cloudpickle in your requirements.
+ - ``"dill"``: Use dill for serialize more complex types,
+ this requires to include dill in your requirements.
+ :param system_site_packages: Whether to include
+ system_site_packages in your virtual environment.
+ If not specified, system_site_packages will be set to ``True` by
default.
+ See virtualenv documentation for more information.
+ :param pip_install_options: a list of pip install options when installing
requirements
+ See 'pip install -h' for available options
+ :param index_urls: an optional list of index urls to load Python packages
from.
+ If not provided the system pip conf will be used to source packages
from.
+ If set to ``[]``, `--no-index` will be used to ignore package index.
+ :param venv_cache_path: Optional path to the virtual environment parent
folder in which the
+ virtual environment will be cached, creates a sub-folder venv-{hash}
whereas hash will be replaced
+ with a checksum of requirements. If not provided the virtual
environment will be created and deleted
+ in a temp folder for every execution.
+ """
+
+ conn_name_attr = "venv_conn_id"
+ default_conn_name = "venv_default"
+ conn_type = "python_venv"
+ hook_name = "Python Virtualenv"
+
+ @classmethod
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
+ v = sys.version_info
+ return {
+ "hidden_fields": ["host", "schema", "port", "login", "password",
"extra"],
+ "relabeling": {},
+ "placeholders": {"python_version": f"{v.major}.{v.minor}"},
+ }
+
+ @classmethod
+ def get_connection_form_widgets(cls) -> dict[str, Any]:
+ from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget,
BS3TextFieldWidget
+ from flask_babel import lazy_gettext
+ from wtforms import BooleanField, StringField
+ from wtforms.validators import InputRequired, Optional
+
+ return {
+ "requirements": StringField(
+ lazy_gettext("Requirements"),
+ description=lazy_gettext("PEP 508 formatted requirements
file."),
+ widget=BS3TextAreaFieldWidget(),
+ validators=[InputRequired()],
+ ),
+ "python_version": StringField(
+ lazy_gettext("Python Version"),
+ description=lazy_gettext("The Python version to run the
virtual environment with."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "system_site_packages": BooleanField(
+ lazy_gettext("System Site Packages"),
+ description=lazy_gettext(
+ "Whether to include system site packages in the virtual
environment."
+ ),
+ default=False,
+ ),
+ "pip_install_options": StringField(
+ lazy_gettext("Pip Install Options"),
+ description=lazy_gettext("Additional options to be passed to
pip install."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "index_urls": StringField(
+ lazy_gettext("Index URLs"),
+ description=lazy_gettext("Comma separated list of URLs to
search for packages."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "venv_cache_path": StringField(
+ lazy_gettext("Virtual Environment Cache Path"),
+ description=lazy_gettext("Path to cache the virtual
environment."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ }
+
+ def __init__(
+ self,
+ venv_conn_id: str | None = default_conn_name,
+ requirements: None | list[str] = None,
+ python_version: str | None = None,
+ system_site_packages: bool | ArgNotSet = NOTSET,
+ pip_install_options: list[str] | None = None,
+ index_urls: None | list[str] = None,
+ venv_cache_path: None | os.PathLike[str] = None,
+ serializer: None | Literal["pickle", "cloudpickle", "dill"] = None,
+ ) -> None:
+ super().__init__()
+
+ self._venv_conn_id = venv_conn_id
+ self._requirements = requirements
+ self._python_version = python_version
+ self._system_site_packages = system_site_packages
+ self._pip_install_options = pip_install_options
+ self._index_urls = index_urls
+ self._venv_cache_path = venv_cache_path
+ self._serializer = serializer
+ self._conn = self.get_connection(venv_conn_id) if venv_conn_id else
None
+
+ def get_conn(self) -> None:
+ pass
+
+ @cached_property
+ def extra(self) -> dict[str, Any]:
+ if self._conn:
+ return self._conn.extra_dejson
+ return {}
Review Comment:
`cached_property` is good to reduce computations upon reuse. Any issue?
##########
airflow/operators/python.py:
##########
@@ -644,10 +646,13 @@ class
PythonVirtualenvOperator(_BasePythonVirtualenvOperator):
exit code will be treated as a failure.
:param index_urls: an optional list of index urls to load Python packages
from.
If not provided the system pip conf will be used to source packages
from.
+ If set to ``[]``, ``--no-index`` will be used to ignore package index.
Review Comment:
It's hard to figure out the fact that if `index_urls=[]`(empty list), it
converted to `--no-index` pip option.
I added comment to help users discover this more quickly.
##########
airflow/hooks/python_virtualenv.py:
##########
@@ -0,0 +1,353 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+import logging
+import os
+import shutil
+import sys
+from contextlib import contextmanager
+from functools import cached_property
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING, Any, Generator
+
+from airflow.exceptions import (
+ AirflowException,
+)
+from airflow.hooks.base import BaseHook
+from airflow.models.variable import Variable
+from airflow.utils import hashlib_wrapper
+from airflow.utils.python_virtualenv import prepare_virtualenv
+from airflow.utils.types import NOTSET, ArgNotSet
+
+if TYPE_CHECKING:
+ from airflow.typing_compat import Literal
+
+log = logging.getLogger(__name__)
+
+
+class PythonVirtualenvHook(BaseHook):
+ """
+ To run a function in a virtualenv, create virtualenv first and destroy
automatically.
+
+ :param venv_conn_id: The ID of Python Virtualenv Connection.
+ If set, the other arguments in the Operator will override the values
from the connection.
+ :param requirements: Either a list of requirement strings, or a (templated)
+ "requirements file" as specified by pip.
+ :param python_version: The Python version to run the virtual environment
with. Note that
+ both 2 and 2.7 are acceptable forms.
+ :param serializer: Which serializer use to serialize the args and result.
It can be one of the following:
+
+ - ``"pickle"``: (default) Use pickle for serialization. Included in
the Python Standard Library.
+ - ``"cloudpickle"``: Use cloudpickle for serialize more complex types,
+ this requires to include cloudpickle in your requirements.
+ - ``"dill"``: Use dill for serialize more complex types,
+ this requires to include dill in your requirements.
+ :param system_site_packages: Whether to include
+ system_site_packages in your virtual environment.
+ If not specified, system_site_packages will be set to ``True` by
default.
+ See virtualenv documentation for more information.
+ :param pip_install_options: a list of pip install options when installing
requirements
+ See 'pip install -h' for available options
+ :param index_urls: an optional list of index urls to load Python packages
from.
+ If not provided the system pip conf will be used to source packages
from.
+ If set to ``[]``, `--no-index` will be used to ignore package index.
+ :param venv_cache_path: Optional path to the virtual environment parent
folder in which the
+ virtual environment will be cached, creates a sub-folder venv-{hash}
whereas hash will be replaced
+ with a checksum of requirements. If not provided the virtual
environment will be created and deleted
+ in a temp folder for every execution.
+ """
+
+ conn_name_attr = "venv_conn_id"
+ default_conn_name = "venv_default"
+ conn_type = "python_venv"
+ hook_name = "Python Virtualenv"
+
+ @classmethod
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
+ v = sys.version_info
+ return {
+ "hidden_fields": ["host", "schema", "port", "login", "password",
"extra"],
+ "relabeling": {},
+ "placeholders": {"python_version": f"{v.major}.{v.minor}"},
+ }
+
+ @classmethod
+ def get_connection_form_widgets(cls) -> dict[str, Any]:
+ from flask_appbuilder.fieldwidgets import BS3TextAreaFieldWidget,
BS3TextFieldWidget
+ from flask_babel import lazy_gettext
+ from wtforms import BooleanField, StringField
+ from wtforms.validators import InputRequired, Optional
+
+ return {
+ "requirements": StringField(
+ lazy_gettext("Requirements"),
+ description=lazy_gettext("PEP 508 formatted requirements
file."),
+ widget=BS3TextAreaFieldWidget(),
+ validators=[InputRequired()],
+ ),
+ "python_version": StringField(
+ lazy_gettext("Python Version"),
+ description=lazy_gettext("The Python version to run the
virtual environment with."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "system_site_packages": BooleanField(
+ lazy_gettext("System Site Packages"),
+ description=lazy_gettext(
+ "Whether to include system site packages in the virtual
environment."
+ ),
+ default=False,
+ ),
+ "pip_install_options": StringField(
+ lazy_gettext("Pip Install Options"),
+ description=lazy_gettext("Additional options to be passed to
pip install."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "index_urls": StringField(
+ lazy_gettext("Index URLs"),
+ description=lazy_gettext("Comma separated list of URLs to
search for packages."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ "venv_cache_path": StringField(
+ lazy_gettext("Virtual Environment Cache Path"),
+ description=lazy_gettext("Path to cache the virtual
environment."),
+ widget=BS3TextFieldWidget(),
+ validators=[Optional()],
+ ),
+ }
+
+ def __init__(
+ self,
+ venv_conn_id: str | None = default_conn_name,
+ requirements: None | list[str] = None,
+ python_version: str | None = None,
+ system_site_packages: bool | ArgNotSet = NOTSET,
+ pip_install_options: list[str] | None = None,
+ index_urls: None | list[str] = None,
+ venv_cache_path: None | os.PathLike[str] = None,
+ serializer: None | Literal["pickle", "cloudpickle", "dill"] = None,
+ ) -> None:
+ super().__init__()
+
+ self._venv_conn_id = venv_conn_id
+ self._requirements = requirements
+ self._python_version = python_version
+ self._system_site_packages = system_site_packages
+ self._pip_install_options = pip_install_options
+ self._index_urls = index_urls
+ self._venv_cache_path = venv_cache_path
+ self._serializer = serializer
+ self._conn = self.get_connection(venv_conn_id) if venv_conn_id else
None
+
+ def get_conn(self) -> None:
+ pass
+
+ @cached_property
+ def extra(self) -> dict[str, Any]:
+ if self._conn:
+ return self._conn.extra_dejson
+ return {}
+
+ @cached_property
+ def requirements(self) -> list[str]:
+ """Extends the requirements from PythonVirtualenvOperator with the
requirements from connection."""
+ requirements = []
+ if self._requirements:
+ requirements.extend([str(requirement) for requirement in
self._requirements])
+ if self.extra.get("requirements"):
+ requirements.extend(
+ [requirement.strip() for requirement in
self.extra["requirements"].split("\n")]
+ )
+ return requirements
Review Comment:
The final requirements combine those defined in the connection with those
input by the user in `PythonVirtualenvOperator`.
##########
airflow/providers_manager.py:
##########
@@ -467,7 +468,7 @@ def _init_airflow_core_hooks(self):
connection_type=None,
connection_testable=False,
)
- for cls in [FSHook, PackageIndexHook]:
+ for cls in [FSHook, PackageIndexHook, PythonVirtualenvHook]:
Review Comment:

The reason the build test is failing is because it fails in
test_views_connection.
https://github.com/apache/airflow/blob/2f0bba012e6808d2dffebcb60f19953c3f6fe959/tests/www/views/test_views_connection.py#L55-L100
In the following code, it goes into Except block.
https://github.com/apache/airflow/blob/2f0bba012e6808d2dffebcb60f19953c3f6fe959/airflow/www/extensions/init_manifest_files.py#L34-L43
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]