This is an automated email from the ASF dual-hosted git repository.

husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new bc1d68a6eb Add a new config for celery result_backend engine options 
(#30426)
bc1d68a6eb is described below

commit bc1d68a6eb01919415c399d678f491e013eb9238
Author: Hussein Awala <[email protected]>
AuthorDate: Fri Jun 2 17:16:25 2023 +0300

    Add a new config for celery result_backend engine options (#30426)
    
    * add a new config for celery result_backend engine options
    
    * add a unit test
    
    * Apply suggestions from code review
    
    Co-authored-by: Niko Oliveira <[email protected]>
    
    * fix static checks
    
    * replace result_backend_engine_options by 
result_backend_sqlalchemy_engine_options
    
    * fix unit tests
    
    * Fix unit test
    
    ---------
    
    Co-authored-by: Niko Oliveira <[email protected]>
---
 airflow/config_templates/config.yml          |  7 +++++++
 airflow/config_templates/default_airflow.cfg |  4 ++++
 airflow/config_templates/default_celery.py   |  3 +++
 tests/executors/test_celery_executor.py      | 18 ++++++++++++++++++
 4 files changed, 32 insertions(+)

diff --git a/airflow/config_templates/config.yml 
b/airflow/config_templates/config.yml
index aeeb18c8e9..3e11c3e117 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -2041,6 +2041,13 @@ celery:
       sensitive: true
       example: "db+postgresql://postgres:airflow@postgres/airflow"
       default: ~
+    result_backend_sqlalchemy_engine_options:
+      description: |
+        Optional configuration dictionary to pass to the Celery result backend 
SQLAlchemy engine.
+      version_added: 2.6.0
+      type: string
+      example: '{"pool_recycle": 1800}'
+      default: ""
     flower_host:
       description: |
         Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
diff --git a/airflow/config_templates/default_airflow.cfg 
b/airflow/config_templates/default_airflow.cfg
index 05bf0a25ad..c28e03bd9b 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -1056,6 +1056,10 @@ broker_url = redis://redis:6379/0
 # Example: result_backend = db+postgresql://postgres:airflow@postgres/airflow
 # result_backend =
 
+# Optional configuration dictionary to pass to the Celery result backend 
SQLAlchemy engine.
+# Example: result_backend_sqlalchemy_engine_options = {{"pool_recycle": 1800}}
+result_backend_sqlalchemy_engine_options =
+
 # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
 # it ``airflow celery flower``. This defines the IP that Celery Flower runs on
 flower_host = 0.0.0.0
diff --git a/airflow/config_templates/default_celery.py 
b/airflow/config_templates/default_celery.py
index 9ec320097e..f3f49ad201 100644
--- a/airflow/config_templates/default_celery.py
+++ b/airflow/config_templates/default_celery.py
@@ -65,6 +65,9 @@ DEFAULT_CELERY_CONFIG = {
     "broker_url": broker_url,
     "broker_transport_options": broker_transport_options_for_celery,
     "result_backend": result_backend,
+    "database_engine_options": conf.getjson(
+        "celery", "result_backend_sqlalchemy_engine_options", fallback={}
+    ),
     "worker_concurrency": conf.getint("celery", "WORKER_CONCURRENCY"),
     "worker_enable_remote_control": conf.getboolean("celery", 
"worker_enable_remote_control"),
 }
diff --git a/tests/executors/test_celery_executor.py 
b/tests/executors/test_celery_executor.py
index 61df13051f..11b461b76d 100644
--- a/tests/executors/test_celery_executor.py
+++ b/tests/executors/test_celery_executor.py
@@ -42,6 +42,7 @@ from airflow.models.taskinstance import TaskInstance, 
TaskInstanceKey
 from airflow.utils import timezone
 from airflow.utils.state import State
 from tests.test_utils import db
+from tests.test_utils.config import conf_vars
 
 FAKE_EXCEPTION_MSG = "Fake Exception"
 
@@ -248,6 +249,23 @@ class TestCeleryExecutor:
         assert app.control.revoke.called_with("231")
         assert mock_fail.called_once()
 
+    @conf_vars({("celery", "result_backend_sqlalchemy_engine_options"): 
'{"pool_recycle": 1800}'})
+    @mock.patch("celery.Celery")
+    def test_result_backend_sqlalchemy_engine_options(self, mock_celery):
+        import importlib
+
+        from airflow.config_templates import default_celery
+        from airflow.executors import celery_executor_utils
+
+        # reload celery conf to apply the new config
+        importlib.reload(default_celery)
+        # reload celery_executor_utils to recreate the celery app with new 
config
+        importlib.reload(celery_executor_utils)
+
+        call_args = mock_celery.call_args.kwargs.get("config_source")
+        assert "database_engine_options" in call_args
+        assert call_args["database_engine_options"] == {"pool_recycle": 1800}
+
 
 def test_operation_timeout_config():
     assert celery_executor_utils.OPERATION_TIMEOUT == 1

Reply via email to