This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cd0a4ed8f88b38a2c6e25ba829f1507860dc23d6
Author: hugo-syn <61210734+hugo-...@users.noreply.github.com>
AuthorDate: Tue Oct 24 12:22:52 2023 +0200
Fix typo s/overriden/overridden/ (#35128)
(cherry picked from commit 6316025b7c160006ef57d3463868394c3f9e8d97)
---
docs/apache-airflow/howto/operator/python.rst | 38 ++++++++++++++++++++++
helm_tests/airflow_aux/test_cleanup_pods.py | 2 +-
helm_tests/airflow_aux/test_create_user_job.py | 2 +-
.../airflow_aux/test_migrate_database_job.py | 2 +-
helm_tests/airflow_core/test_dag_processor.py | 2 +-
helm_tests/airflow_core/test_scheduler.py | 2 +-
helm_tests/airflow_core/test_triggerer.py | 2 +-
helm_tests/airflow_core/test_worker.py | 2 +-
helm_tests/other/test_flower.py | 2 +-
helm_tests/other/test_pgbouncer.py | 2 +-
helm_tests/other/test_redis.py | 2 +-
helm_tests/other/test_statsd.py | 2 +-
helm_tests/webserver/test_webserver.py | 2 +-
tests/decorators/test_setup_teardown.py | 6 ++--
tests/decorators/test_task_group.py | 2 +-
15 files changed, 54 insertions(+), 16 deletions(-)
diff --git a/docs/apache-airflow/howto/operator/python.rst
b/docs/apache-airflow/howto/operator/python.rst
index e2bf4b11e5..7e9b9625e3 100644
--- a/docs/apache-airflow/howto/operator/python.rst
+++ b/docs/apache-airflow/howto/operator/python.rst
@@ -112,6 +112,44 @@ If additional parameters for package installation are
needed pass them in ``requ
All supported options are listed in the `requirements file format
<https://pip.pypa.io/en/stable/reference/requirements-file-format/#supported-options>`_.
+<<<<<<< HEAD
+=======
+Virtual environment setup options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The virtual environment is created based on the global python pip
configuration on your worker. Using additional ENVs in your environment or
adjustments in the general
+pip configuration as described in `pip config
<https://pip.pypa.io/en/stable/topics/configuration/>`_.
+
+If you want to use additional task specific private python repositories to
setup the virtual environment, you can pass the ``index_urls`` parameter which
will adjust the
+pip install configurations. Passed index urls replace the standard system
configured index url settings.
+To prevent adding secrets to the private repository in your DAG code you can
use the Airflow
+:doc:`../../authoring-and-scheduling/connections`. For this purpose the
connection type ``Package Index (Python)`` can be used.
+
+In the special case you want to prevent remote calls for setup of a virtual
environment, pass the ``index_urls`` as empty list as ``index_urls=[]`` which
+forced pip installer to use the ``--no-index`` option.
+
+Caching and reuse
+^^^^^^^^^^^^^^^^^
+
+Setup of virtual environments is made per task execution in a temporary
directory. After execution the virtual environment is deleted again. Ensure
that the ``$tmp`` folder
+on your workers have sufficient disk space. Usually (if not configured
differently) the local pip cache will be used preventing a re-download of
packages
+for each execution.
+
+But still setting up the virtual environment for every execution needs some
time. For repeated execution you can set the option ``venv_cache_path`` to a
file system
+folder on your worker. In this case the virtual environment will be set up
once and be re-used. If virtual environment caching is used, per unique
requirements set different
+virtual environment subfolders are created in the cache path. So depending on
your variations in the DAGs in your system setup sufficient disk space is
needed.
+
+Note that no automated cleanup is made and in case of cached mode. All worker
slots share the same virtual environment but if tasks are scheduled over and
over on
+different workers, it might happen that virtual environment are created on
multiple workers individually. Also if the worker is started in a Kubernetes
POD, a restart
+of the worker will drop the cache (assuming ``venv_cache_path`` is not on a
persistent volume).
+
+In case you have problems during runtime with broken cached virtual
environments, you can influence the cache directory hash by setting the Airflow
variable
+``PythonVirtualenvOperator.cache_key`` to any text. The content of this
variable is uses in the vector to calculate the cache directory key.
+
+Note that any modification of a cached virtual environment (like temp files in
binary path, post-installing further requirements) might pollute a cached
virtual environment and the
+operator is not maintaining or cleaning the cache path.
+
+>>>>>>> ff25411980 (Fix spellcheck error. (#35041))
.. _howto/operator:ExternalPythonOperator:
diff --git a/helm_tests/airflow_aux/test_cleanup_pods.py
b/helm_tests/airflow_aux/test_cleanup_pods.py
index 33e475d59f..79563aee4a 100644
--- a/helm_tests/airflow_aux/test_cleanup_pods.py
+++ b/helm_tests/airflow_aux/test_cleanup_pods.py
@@ -373,7 +373,7 @@ class TestCleanupServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"cleanup": {"enabled": True, "serviceAccount":
{"automountServiceAccountToken": False}},
diff --git a/helm_tests/airflow_aux/test_create_user_job.py
b/helm_tests/airflow_aux/test_create_user_job.py
index 3ed0cf04af..bb85a7d3cf 100644
--- a/helm_tests/airflow_aux/test_create_user_job.py
+++ b/helm_tests/airflow_aux/test_create_user_job.py
@@ -408,7 +408,7 @@ class TestCreateUserJobServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"createUserJob": {
diff --git a/helm_tests/airflow_aux/test_migrate_database_job.py
b/helm_tests/airflow_aux/test_migrate_database_job.py
index 6bd52a1100..27f86c8dbb 100644
--- a/helm_tests/airflow_aux/test_migrate_database_job.py
+++ b/helm_tests/airflow_aux/test_migrate_database_job.py
@@ -339,7 +339,7 @@ class TestMigrateDatabaseJobServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"migrateDatabaseJob": {
diff --git a/helm_tests/airflow_core/test_dag_processor.py
b/helm_tests/airflow_core/test_dag_processor.py
index 74035222e7..4ba7ebf28a 100644
--- a/helm_tests/airflow_core/test_dag_processor.py
+++ b/helm_tests/airflow_core/test_dag_processor.py
@@ -641,7 +641,7 @@ class TestDagProcessorServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"dagProcessor": {
diff --git a/helm_tests/airflow_core/test_scheduler.py
b/helm_tests/airflow_core/test_scheduler.py
index 57ab3bbe5b..08971dea69 100644
--- a/helm_tests/airflow_core/test_scheduler.py
+++ b/helm_tests/airflow_core/test_scheduler.py
@@ -815,7 +815,7 @@ class TestSchedulerServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"scheduler": {
diff --git a/helm_tests/airflow_core/test_triggerer.py
b/helm_tests/airflow_core/test_triggerer.py
index 64f1a3419b..10366a219b 100644
--- a/helm_tests/airflow_core/test_triggerer.py
+++ b/helm_tests/airflow_core/test_triggerer.py
@@ -602,7 +602,7 @@ class TestTriggererServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"triggerer": {
diff --git a/helm_tests/airflow_core/test_worker.py
b/helm_tests/airflow_core/test_worker.py
index 81432ee7ac..a93f450dc0 100644
--- a/helm_tests/airflow_core/test_worker.py
+++ b/helm_tests/airflow_core/test_worker.py
@@ -814,7 +814,7 @@ class TestWorkerServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"workers": {
diff --git a/helm_tests/other/test_flower.py b/helm_tests/other/test_flower.py
index e195d146e7..f48ac1501d 100644
--- a/helm_tests/other/test_flower.py
+++ b/helm_tests/other/test_flower.py
@@ -592,7 +592,7 @@ class TestFlowerServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"flower": {
diff --git a/helm_tests/other/test_pgbouncer.py
b/helm_tests/other/test_pgbouncer.py
index 454912b4fb..122057762b 100644
--- a/helm_tests/other/test_pgbouncer.py
+++ b/helm_tests/other/test_pgbouncer.py
@@ -584,7 +584,7 @@ class TestPgBouncerServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"pgbouncer": {
diff --git a/helm_tests/other/test_redis.py b/helm_tests/other/test_redis.py
index c99503a4b7..51a84dc532 100644
--- a/helm_tests/other/test_redis.py
+++ b/helm_tests/other/test_redis.py
@@ -344,7 +344,7 @@ class TestRedisServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"redis": {
diff --git a/helm_tests/other/test_statsd.py b/helm_tests/other/test_statsd.py
index c383cd99d3..bf3737e6bf 100644
--- a/helm_tests/other/test_statsd.py
+++ b/helm_tests/other/test_statsd.py
@@ -311,7 +311,7 @@ class TestStatsdServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"statsd": {
diff --git a/helm_tests/webserver/test_webserver.py
b/helm_tests/webserver/test_webserver.py
index d76cdb2eaa..8b9a58058e 100644
--- a/helm_tests/webserver/test_webserver.py
+++ b/helm_tests/webserver/test_webserver.py
@@ -1026,7 +1026,7 @@ class TestWebserverServiceAccount:
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is True
- def test_overriden_automount_service_account_token(self):
+ def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
"webserver": {
diff --git a/tests/decorators/test_setup_teardown.py
b/tests/decorators/test_setup_teardown.py
index 4ec8376a07..281b97f8c6 100644
--- a/tests/decorators/test_setup_teardown.py
+++ b/tests/decorators/test_setup_teardown.py
@@ -186,7 +186,7 @@ class TestSetupTearDownTask:
assert teardown_task.on_failure_fail_dagrun is on_failure_fail_dagrun
assert len(dag.task_group.children) == 1
- def test_setup_task_can_be_overriden(self, dag_maker):
+ def test_setup_task_can_be_overridden(self, dag_maker):
@setup
def mytask():
print("I am a setup task")
@@ -197,7 +197,7 @@ class TestSetupTearDownTask:
setup_task = dag.task_group.children["mytask2"]
assert setup_task.is_setup
- def test_teardown_on_failure_fail_dagrun_can_be_overriden(self, dag_maker):
+ def test_teardown_on_failure_fail_dagrun_can_be_overridden(self,
dag_maker):
@teardown
def mytask():
print("I am a teardown task")
@@ -209,7 +209,7 @@ class TestSetupTearDownTask:
assert teardown_task.is_teardown
assert teardown_task.on_failure_fail_dagrun
- def
test_retain_on_failure_fail_dagrun_when_other_attrs_are_overriden(self,
dag_maker):
+ def
test_retain_on_failure_fail_dagrun_when_other_attrs_are_overridden(self,
dag_maker):
@teardown(on_failure_fail_dagrun=True)
def mytask():
print("I am a teardown task")
diff --git a/tests/decorators/test_task_group.py
b/tests/decorators/test_task_group.py
index 4c741ef1c1..632782c54e 100644
--- a/tests/decorators/test_task_group.py
+++ b/tests/decorators/test_task_group.py
@@ -77,7 +77,7 @@ def test_tooltip_derived_from_function_docstring():
assert _.task_group_dict["tg"].tooltip == "Function docstring."
-def test_tooltip_not_overriden_by_function_docstring():
+def test_tooltip_not_overridden_by_function_docstring():
"""
Test that the tooltip for TaskGroup is the explicitly set value even if
the decorated function has a
docstring.
