This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 85d1d1f2514 add gc.freeze to minimize memory usage in celery worker
(#62212)
85d1d1f2514 is described below
commit 85d1d1f25149ac424661df708e1c6d6d55a029f6
Author: Jeongwoo Do <[email protected]>
AuthorDate: Thu Mar 12 10:22:12 2026 +0900
add gc.freeze to minimize memory usage in celery worker (#62212)
---
.../providers/celery/executors/celery_executor_utils.py | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git
a/providers/celery/src/airflow/providers/celery/executors/celery_executor_utils.py
b/providers/celery/src/airflow/providers/celery/executors/celery_executor_utils.py
index 578d0a909ac..44195f2475e 100644
---
a/providers/celery/src/airflow/providers/celery/executors/celery_executor_utils.py
+++
b/providers/celery/src/airflow/providers/celery/executors/celery_executor_utils.py
@@ -23,6 +23,7 @@ Much of this code is expensive to import/load, be careful
where this module is i
from __future__ import annotations
import contextlib
+import gc
import logging
import math
import os
@@ -37,7 +38,7 @@ from typing import TYPE_CHECKING, Any
from celery import Celery, states as celery_states
from celery.backends.base import BaseKeyValueStoreBackend
from celery.backends.database import DatabaseBackend, Task as TaskDb, retry,
session_cleanup
-from celery.signals import import_modules as celery_import_modules
+from celery.signals import import_modules as celery_import_modules,
worker_ready
from sqlalchemy import select
from airflow.configuration import AirflowConfigParser, conf
@@ -174,6 +175,15 @@ def on_celery_import_modules(*args, **kwargs):
with contextlib.suppress(ImportError):
import kubernetes.client # noqa: F401
+ # To prevent memory increase by COW in celery's ForkPoolWorker.
+ gc.freeze()
+
+
+@worker_ready.connect
+def on_celery_worker_ready(*args, **kwargs):
+ # Unfreeze the objects from gc freeze when the ForkPoolWorker is all
loaded.
+ gc.unfreeze()
+
# Once Celery 5.5 is out of beta, we can pass `pydantic=True` to the decorator
and it will handle the validation
# and deserialization for us
