This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6ffe86525057806dd4f4e7846e33c007abe0ecec
Author: Daniel Standish <15932138+dstand...@users.noreply.github.com>
AuthorDate: Wed Oct 4 10:29:04 2023 -0700
Remove warning about max_tis per query > parallelism (#34742)
This warning is erroneous because it's not true in all cases that the max
tis per query setting is disregarded when greater than parallelism.
See discussion at
https://github.com/apache/airflow/pull/32572#issuecomment-1739787276.
(cherry picked from commit 49bb45ca8662fff9136fc218d9e242bdce1a0afe)
---
airflow/configuration.py | 21 ---------------------
tests/core/test_configuration.py | 16 ----------------
2 files changed, 37 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 007f9cc5c4..ccd3f5957a 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -721,7 +721,6 @@ class AirflowConfigParser(ConfigParser):
def validate(self):
self._validate_sqlite3_version()
self._validate_enums()
- self._validate_max_tis_per_query()
for section, replacement in self.deprecated_values.items():
for name, info in replacement.items():
@@ -743,26 +742,6 @@ class AirflowConfigParser(ConfigParser):
self._upgrade_postgres_metastore_conn()
self.is_validated = True
- def _validate_max_tis_per_query(self) -> None:
- """
- Check if config ``scheduler.max_tis_per_query`` is not greater than
``core.parallelism``.
-
- If not met, a warning message is printed to guide the user to correct
it.
-
- More info: https://github.com/apache/airflow/pull/32572
- """
- max_tis_per_query = self.getint("scheduler", "max_tis_per_query")
- parallelism = self.getint("core", "parallelism")
-
- if max_tis_per_query > parallelism:
- warnings.warn(
- f"Config scheduler.max_tis_per_query (value:
{max_tis_per_query}) "
- f"should NOT be greater than core.parallelism (value:
{parallelism}). "
- "Will now use core.parallelism as the max task instances per
query "
- "instead of specified value.",
- UserWarning,
- )
-
def _upgrade_auth_backends(self):
"""
Ensure a custom auth_backends setting contains session.
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index 103e8929b9..c61eaf58d4 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -766,22 +766,6 @@ notacommand = OK
)
assert message == exception
- @mock.patch.dict(
- "os.environ",
- {
- "AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY": "200",
- "AIRFLOW__CORE__PARALLELISM": "100",
- },
- )
- def test_max_tis_per_query_too_high(self):
- test_conf = AirflowConfigParser()
-
- with pytest.warns(UserWarning) as ctx:
- test_conf._validate_max_tis_per_query()
-
- captured_warnings_msg = str(ctx.pop().message)
- assert "max_tis_per_query" in captured_warnings_msg and
"core.parallelism" in captured_warnings_msg
-
def test_as_dict_works_without_sensitive_cmds(self):
conf_materialize_cmds = conf.as_dict(display_sensitive=True, raw=True,
include_cmds=True)
conf_maintain_cmds = conf.as_dict(display_sensitive=True, raw=True,
include_cmds=False)
