This is an automated email from the ASF dual-hosted git repository. dstandish pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 49bb45ca86 Remove warning about max_tis per query > parallelism (#34742) 49bb45ca86 is described below commit 49bb45ca8662fff9136fc218d9e242bdce1a0afe Author: Daniel Standish <15932138+dstand...@users.noreply.github.com> AuthorDate: Wed Oct 4 10:29:04 2023 -0700 Remove warning about max_tis per query > parallelism (#34742) This warning is erroneous because it's not true in all cases that the max tis per query setting is disregarded when greater than parallelism. See discussion at https://github.com/apache/airflow/pull/32572#issuecomment-1739787276. --- airflow/configuration.py | 21 --------------------- tests/core/test_configuration.py | 16 ---------------- 2 files changed, 37 deletions(-) diff --git a/airflow/configuration.py b/airflow/configuration.py index 0107cb1021..ab26bbbecf 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -717,7 +717,6 @@ class AirflowConfigParser(ConfigParser): def validate(self): self._validate_sqlite3_version() self._validate_enums() - self._validate_max_tis_per_query() for section, replacement in self.deprecated_values.items(): for name, info in replacement.items(): @@ -739,26 +738,6 @@ class AirflowConfigParser(ConfigParser): self._upgrade_postgres_metastore_conn() self.is_validated = True - def _validate_max_tis_per_query(self) -> None: - """ - Check if config ``scheduler.max_tis_per_query`` is not greater than ``core.parallelism``. - - If not met, a warning message is printed to guide the user to correct it. - - More info: https://github.com/apache/airflow/pull/32572 - """ - max_tis_per_query = self.getint("scheduler", "max_tis_per_query") - parallelism = self.getint("core", "parallelism") - - if max_tis_per_query > parallelism: - warnings.warn( - f"Config scheduler.max_tis_per_query (value: {max_tis_per_query}) " - f"should NOT be greater than core.parallelism (value: {parallelism}). " - "Will now use core.parallelism as the max task instances per query " - "instead of specified value.", - UserWarning, - ) - def _upgrade_auth_backends(self): """ Ensure a custom auth_backends setting contains session. diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 2c6ad3ecfa..8137f1031d 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -760,22 +760,6 @@ notacommand = OK ) assert message == exception - @mock.patch.dict( - "os.environ", - { - "AIRFLOW__SCHEDULER__MAX_TIS_PER_QUERY": "200", - "AIRFLOW__CORE__PARALLELISM": "100", - }, - ) - def test_max_tis_per_query_too_high(self): - test_conf = AirflowConfigParser() - - with pytest.warns(UserWarning) as ctx: - test_conf._validate_max_tis_per_query() - - captured_warnings_msg = str(ctx.pop().message) - assert "max_tis_per_query" in captured_warnings_msg and "core.parallelism" in captured_warnings_msg - def test_as_dict_works_without_sensitive_cmds(self): conf_materialize_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=True) conf_maintain_cmds = conf.as_dict(display_sensitive=True, raw=True, include_cmds=False)