ashb commented on code in PR #46265:
URL: https://github.com/apache/airflow/pull/46265#discussion_r1934764277
##########
providers/celery/src/airflow/providers/celery/executors/celery_executor_utils.py:
##########
@@ -125,21 +126,54 @@ def on_celery_import_modules(*args, **kwargs):
import kubernetes.client # noqa: F401
[email protected]
-def execute_command(command_to_exec: CommandType) -> None:
- """Execute command."""
- dag_id, task_id =
BaseExecutor.validate_airflow_tasks_run_command(command_to_exec)
+# Once Celery5 is out of beta, we can pass `pydantic=True` to the decorator
and it will handle the validation
+# and deserialization for us
[email protected](name="execute_workload")
+def execute_workload(input: str) -> None:
+ from pydantic import TypeAdapter
+
+ from airflow.configuration import conf
+ from airflow.executors import workloads
+ from airflow.sdk.execution_time.supervisor import supervise
+
+ decoder = TypeAdapter(workloads.All)
+ workload = decoder.validate_json(input)
+
celery_task_id = app.current_task.request.id
- log.info("[%s] Executing command in Celery: %s", celery_task_id,
command_to_exec)
- with _airflow_parsing_context_manager(dag_id=dag_id, task_id=task_id):
- try:
- if settings.EXECUTE_TASKS_NEW_PYTHON_INTERPRETER:
- _execute_in_subprocess(command_to_exec, celery_task_id)
- else:
- _execute_in_fork(command_to_exec, celery_task_id)
- except Exception:
- Stats.incr("celery.execute_command.failure")
- raise
+
+ if not isinstance(workload, workloads.ExecuteTask):
+ raise ValueError(f"CeleryExecutor does not now how to handle
{type(workload)}")
+
+ log.info("[%s] Executing workload in Celery: %s", celery_task_id, workload)
+
+ supervise(
+ # This is the "wrong" ti type, but it duck types the same. TODO:
Create a protocol for this.
+ ti=workload.ti, # type: ignore[arg-type]
+ dag_rel_path=workload.dag_rel_path,
+ bundle_info=workload.bundle_info,
+ token=workload.token,
+ server=conf.get("workers", "execution_api_server_url",
fallback="http://localhost:9091/execution/";),
+ log_path=workload.log_path,
+ )
+
+
+if not AIRFLOW_V_3_0_PLUS:
+
+ @app.task
+ def execute_command(command_to_exec: CommandType) -> None:
Review Comment:
Note: as mentioned in the PR description, when running on Airflow 3, we
don't register the old task anymore
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
