This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bc1d68a6eb Add a new config for celery result_backend engine options
(#30426)
bc1d68a6eb is described below
commit bc1d68a6eb01919415c399d678f491e013eb9238
Author: Hussein Awala <[email protected]>
AuthorDate: Fri Jun 2 17:16:25 2023 +0300
Add a new config for celery result_backend engine options (#30426)
* add a new config for celery result_backend engine options
* add a unit test
* Apply suggestions from code review
Co-authored-by: Niko Oliveira <[email protected]>
* fix static checks
* replace result_backend_engine_options by
result_backend_sqlalchemy_engine_options
* fix unit tests
* Fix unit test
---------
Co-authored-by: Niko Oliveira <[email protected]>
---
airflow/config_templates/config.yml | 7 +++++++
airflow/config_templates/default_airflow.cfg | 4 ++++
airflow/config_templates/default_celery.py | 3 +++
tests/executors/test_celery_executor.py | 18 ++++++++++++++++++
4 files changed, 32 insertions(+)
diff --git a/airflow/config_templates/config.yml
b/airflow/config_templates/config.yml
index aeeb18c8e9..3e11c3e117 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -2041,6 +2041,13 @@ celery:
sensitive: true
example: "db+postgresql://postgres:airflow@postgres/airflow"
default: ~
+ result_backend_sqlalchemy_engine_options:
+ description: |
+ Optional configuration dictionary to pass to the Celery result backend
SQLAlchemy engine.
+ version_added: 2.6.0
+ type: string
+ example: '{"pool_recycle": 1800}'
+ default: ""
flower_host:
description: |
Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
diff --git a/airflow/config_templates/default_airflow.cfg
b/airflow/config_templates/default_airflow.cfg
index 05bf0a25ad..c28e03bd9b 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -1056,6 +1056,10 @@ broker_url = redis://redis:6379/0
# Example: result_backend = db+postgresql://postgres:airflow@postgres/airflow
# result_backend =
+# Optional configuration dictionary to pass to the Celery result backend
SQLAlchemy engine.
+# Example: result_backend_sqlalchemy_engine_options = {{"pool_recycle": 1800}}
+result_backend_sqlalchemy_engine_options =
+
# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
# it ``airflow celery flower``. This defines the IP that Celery Flower runs on
flower_host = 0.0.0.0
diff --git a/airflow/config_templates/default_celery.py
b/airflow/config_templates/default_celery.py
index 9ec320097e..f3f49ad201 100644
--- a/airflow/config_templates/default_celery.py
+++ b/airflow/config_templates/default_celery.py
@@ -65,6 +65,9 @@ DEFAULT_CELERY_CONFIG = {
"broker_url": broker_url,
"broker_transport_options": broker_transport_options_for_celery,
"result_backend": result_backend,
+ "database_engine_options": conf.getjson(
+ "celery", "result_backend_sqlalchemy_engine_options", fallback={}
+ ),
"worker_concurrency": conf.getint("celery", "WORKER_CONCURRENCY"),
"worker_enable_remote_control": conf.getboolean("celery",
"worker_enable_remote_control"),
}
diff --git a/tests/executors/test_celery_executor.py
b/tests/executors/test_celery_executor.py
index 61df13051f..11b461b76d 100644
--- a/tests/executors/test_celery_executor.py
+++ b/tests/executors/test_celery_executor.py
@@ -42,6 +42,7 @@ from airflow.models.taskinstance import TaskInstance,
TaskInstanceKey
from airflow.utils import timezone
from airflow.utils.state import State
from tests.test_utils import db
+from tests.test_utils.config import conf_vars
FAKE_EXCEPTION_MSG = "Fake Exception"
@@ -248,6 +249,23 @@ class TestCeleryExecutor:
assert app.control.revoke.called_with("231")
assert mock_fail.called_once()
+ @conf_vars({("celery", "result_backend_sqlalchemy_engine_options"):
'{"pool_recycle": 1800}'})
+ @mock.patch("celery.Celery")
+ def test_result_backend_sqlalchemy_engine_options(self, mock_celery):
+ import importlib
+
+ from airflow.config_templates import default_celery
+ from airflow.executors import celery_executor_utils
+
+ # reload celery conf to apply the new config
+ importlib.reload(default_celery)
+ # reload celery_executor_utils to recreate the celery app with new
config
+ importlib.reload(celery_executor_utils)
+
+ call_args = mock_celery.call_args.kwargs.get("config_source")
+ assert "database_engine_options" in call_args
+ assert call_args["database_engine_options"] == {"pool_recycle": 1800}
+
def test_operation_timeout_config():
assert celery_executor_utils.OPERATION_TIMEOUT == 1