This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 6cde25f1ac0 Move `[scheduler]dag_dir_list_interval` to `[dag_bundles]refresh_interval` (#45722) 6cde25f1ac0 is described below commit 6cde25f1ac005e217b32f19a8bf1be75bf8e7af6 Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com> AuthorDate: Fri Jan 17 01:38:48 2025 -0700 Move `[scheduler]dag_dir_list_interval` to `[dag_bundles]refresh_interval` (#45722) This config should be the default refresh_interval in AF3, so move its predecesor. --- airflow/config_templates/config.yml | 16 ++++++++-------- airflow/config_templates/unit_tests.cfg | 1 - airflow/configuration.py | 4 +++- airflow/dag_processing/bundles/base.py | 8 +++++++- .../administration-and-deployment/scheduler.rst | 3 --- docs/apache-airflow/best-practices.rst | 2 +- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 55d4a9fbeb2..ba6af6ca11e 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -2286,13 +2286,6 @@ scheduler: type: integer example: ~ default: "50" - dag_dir_list_interval: - description: | - How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. - version_added: ~ - type: integer - example: ~ - default: "300" print_stats_interval: description: | How often should stats be printed to the logs. Setting to 0 will disable printing stats @@ -2681,7 +2674,7 @@ dag_bundles: description: | List of backend configs. Must supply name, classpath, and kwargs for each backend. - By default, ``refresh_interval`` is set to ``[scheduler] dag_dir_list_interval``, but that can + By default, ``refresh_interval`` is set to ``[dag_bundles] refresh_interval``, but that can also be overridden in kwargs if desired. The default is the dags folder dag bundle. @@ -2711,3 +2704,10 @@ dag_bundles: "kwargs": {{}} }} ] + refresh_interval: + description: | + How often (in seconds) to refresh, or look for new files, in a DAG bundle. + version_added: ~ + type: integer + example: ~ + default: "300" diff --git a/airflow/config_templates/unit_tests.cfg b/airflow/config_templates/unit_tests.cfg index b29c642afe7..9e222ce5504 100644 --- a/airflow/config_templates/unit_tests.cfg +++ b/airflow/config_templates/unit_tests.cfg @@ -88,7 +88,6 @@ result_backend = db+mysql://airflow:airflow@localhost:3306/airflow job_heartbeat_sec = 1 scheduler_heartbeat_sec = 5 parsing_processes = 2 -dag_dir_list_interval = 0 [triggerer] # Those values are set so that during unit tests things run faster than usual. diff --git a/airflow/configuration.py b/airflow/configuration.py index cb0cb5af73a..a0822a55278 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -324,7 +324,9 @@ class AirflowConfigParser(ConfigParser): # A mapping of (new section, new option) -> (old section, old option, since_version). # When reading new option, the old option will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old option will be used instead - deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = {} + deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = { + ("dag_bundles", "refresh_interval"): ("scheduler", "dag_dir_list_interval", "3.0"), + } # A mapping of new section -> (old section, since_version). deprecated_sections: dict[str, tuple[str, str]] = {} diff --git a/airflow/dag_processing/bundles/base.py b/airflow/dag_processing/bundles/base.py index 5ec1e9f5c34..da60f77cf4a 100644 --- a/airflow/dag_processing/bundles/base.py +++ b/airflow/dag_processing/bundles/base.py @@ -46,7 +46,13 @@ class BaseDagBundle(ABC): supports_versioning: bool = False - def __init__(self, *, name: str, refresh_interval: int = 300, version: str | None = None) -> None: + def __init__( + self, + *, + name: str, + refresh_interval: int = conf.getint("dag_bundles", "refresh_interval"), + version: str | None = None, + ) -> None: self.name = name self.version = version self.refresh_interval = refresh_interval diff --git a/docs/apache-airflow/administration-and-deployment/scheduler.rst b/docs/apache-airflow/administration-and-deployment/scheduler.rst index 476e2e66279..d3f05738e11 100644 --- a/docs/apache-airflow/administration-and-deployment/scheduler.rst +++ b/docs/apache-airflow/administration-and-deployment/scheduler.rst @@ -354,9 +354,6 @@ However, you can also look at other non-performance-related scheduler configurat queued tasks that were launched by the dead process will be "adopted" and monitored by this scheduler instead. -- :ref:`config:scheduler__dag_dir_list_interval` - How often (in seconds) to scan the DAGs directory for new files. - - :ref:`config:scheduler__file_parsing_sort_mode` The scheduler will list and sort the DAG files to decide the parsing order. diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst index 37092f31a6d..4b25ea262ad 100644 --- a/docs/apache-airflow/best-practices.rst +++ b/docs/apache-airflow/best-practices.rst @@ -443,7 +443,7 @@ each parameter by following the links): * :ref:`config:scheduler__scheduler_idle_sleep_time` * :ref:`config:scheduler__min_file_process_interval` -* :ref:`config:scheduler__dag_dir_list_interval` +* :ref:`config:dag_bundles__refresh_interval` * :ref:`config:scheduler__parsing_processes` * :ref:`config:scheduler__file_parsing_sort_mode`