This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fcbbf47864 [AIP-51] Executors vending CLI commands (#29055)
fcbbf47864 is described below
commit fcbbf47864c251046de108aafdad394d66e1df23
Author: Niko Oliveira <[email protected]>
AuthorDate: Fri Jul 28 23:36:05 2023 -0700
[AIP-51] Executors vending CLI commands (#29055)
* Base implementation of executor vending cli commands
Update the cli_parser construction (and associated tests) to now get cli
methods from the executor modules.
* Move existing Celery and Kubernetes cli commands to their module
Move the existing cli commands out of the base/core cli module to the
respective executor modules
---
airflow/cli/cli_config.py | 151 -------------------
airflow/cli/cli_parser.py | 18 +++
airflow/executors/base_executor.py | 10 ++
airflow/executors/executor_loader.py | 16 +-
.../providers/celery/executors/celery_executor.py | 161 +++++++++++++++++++++
.../celery/executors/celery_kubernetes_executor.py | 4 +
.../kubernetes/executors/kubernetes_executor.py | 82 +++++++++++
.../executors/local_kubernetes_executor.py | 4 +
airflow/stats.py | 7 +-
tests/cli/conftest.py | 12 ++
tests/cli/test_cli_parser.py | 55 +++++--
tests/executors/test_base_executor.py | 4 +
.../cli/commands/test_celery_command.py | 7 +-
.../celery/executors/test_celery_executor.py | 3 +
.../executors/test_celery_kubernetes_executor.py | 3 +
.../executors/test_kubernetes_executor.py | 3 +
.../executors/test_local_kubernetes_executor.py | 3 +
17 files changed, 370 insertions(+), 173 deletions(-)
diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py
index 8e4c984450..02908ee8f0 100644
--- a/airflow/cli/cli_config.py
+++ b/airflow/cli/cli_config.py
@@ -24,7 +24,6 @@ import argparse
import json
import os
import textwrap
-from argparse import ArgumentError
from typing import Callable, Iterable, NamedTuple, Union
import lazy_object_proxy
@@ -32,8 +31,6 @@ import lazy_object_proxy
from airflow import settings
from airflow.cli.commands.legacy_commands import check_legacy_command
from airflow.configuration import conf
-from airflow.executors.executor_constants import CELERY_EXECUTOR,
CELERY_KUBERNETES_EXECUTOR
-from airflow.executors.executor_loader import ExecutorLoader
from airflow.settings import _ENABLE_AIP_44
from airflow.utils.cli import ColorMode
from airflow.utils.module_loading import import_string
@@ -61,46 +58,6 @@ class DefaultHelpParser(argparse.ArgumentParser):
def _check_value(self, action, value):
"""Override _check_value and check conditionally added command."""
- if action.dest == "subcommand" and value == "celery":
- executor = conf.get("core", "EXECUTOR")
- if executor not in (CELERY_EXECUTOR, CELERY_KUBERNETES_EXECUTOR):
- executor_cls, _ = ExecutorLoader.import_executor_cls(executor)
- classes = ()
- try:
- from airflow.providers.celery.executors.celery_executor
import CeleryExecutor
-
- classes += (CeleryExecutor,)
- except ImportError:
- message = (
- "The celery subcommand requires that you pip install
the celery module. "
- "To do it, run: pip install 'apache-airflow[celery]'"
- )
- raise ArgumentError(action, message)
- try:
- from
airflow.providers.celery.executors.celery_kubernetes_executor import (
- CeleryKubernetesExecutor,
- )
-
- classes += (CeleryKubernetesExecutor,)
- except ImportError:
- pass
- if not issubclass(executor_cls, classes):
- message = (
- f"celery subcommand works only with CeleryExecutor,
CeleryKubernetesExecutor and "
- f"executors derived from them, your current executor:
{executor}, subclassed from: "
- f'{", ".join([base_cls.__qualname__ for base_cls in
executor_cls.__bases__])}'
- )
- raise ArgumentError(action, message)
- if action.dest == "subcommand" and value == "kubernetes":
- try:
- import kubernetes.client # noqa: F401
- except ImportError:
- message = (
- "The kubernetes subcommand requires that you pip install
the kubernetes python client. "
- "To do it, run: pip install
'apache-airflow[cncf.kubernetes]'"
- )
- raise ArgumentError(action, message)
-
if action.choices is not None and value not in action.choices:
check_legacy_command(action, value)
@@ -823,25 +780,6 @@ ARG_DO_PICKLE = Arg(
action="store_true",
)
-ARG_QUEUES = Arg(
- ("-q", "--queues"),
- help="Comma delimited list of queues to serve",
- default=conf.get("operators", "DEFAULT_QUEUE"),
-)
-ARG_CONCURRENCY = Arg(
- ("-c", "--concurrency"),
- type=int,
- help="The number of worker processes",
- default=conf.getint("celery", "worker_concurrency"),
-)
-ARG_CELERY_HOSTNAME = Arg(
- ("-H", "--celery-hostname"),
- help="Set the hostname of celery worker if you have multiple workers on a
single machine",
-)
-ARG_UMASK = Arg(
- ("-u", "--umask"),
- help="Set the umask of celery worker in daemon mode",
-)
ARG_WITHOUT_MINGLE = Arg(
("--without-mingle",),
default=False,
@@ -855,34 +793,6 @@ ARG_WITHOUT_GOSSIP = Arg(
action="store_true",
)
-# flower
-ARG_BROKER_API = Arg(("-a", "--broker-api"), help="Broker API")
-ARG_FLOWER_HOSTNAME = Arg(
- ("-H", "--hostname"),
- default=conf.get("celery", "FLOWER_HOST"),
- help="Set the hostname on which to run the server",
-)
-ARG_FLOWER_PORT = Arg(
- ("-p", "--port"),
- default=conf.getint("celery", "FLOWER_PORT"),
- type=int,
- help="The port on which to run the server",
-)
-ARG_FLOWER_CONF = Arg(("-c", "--flower-conf"), help="Configuration file for
flower")
-ARG_FLOWER_URL_PREFIX = Arg(
- ("-u", "--url-prefix"),
- default=conf.get("celery", "FLOWER_URL_PREFIX"),
- help="URL prefix for Flower",
-)
-ARG_FLOWER_BASIC_AUTH = Arg(
- ("-A", "--basic-auth"),
- default=conf.get("celery", "FLOWER_BASIC_AUTH"),
- help=(
- "Securing Flower with Basic Authentication. "
- "Accepts user:password pairs separated by a comma. "
- "Example: flower_basic_auth = user1:password1,user2:password2"
- ),
-)
ARG_TASK_PARAMS = Arg(("-t", "--task-params"), help="Sends a JSON params dict
to the task")
ARG_POST_MORTEM = Arg(
("-m", "--post-mortem"), action="store_true", help="Open debugger on
uncaught exception"
@@ -1978,55 +1888,6 @@ ROLES_COMMANDS = (
),
)
-CELERY_COMMANDS = (
- ActionCommand(
- name="worker",
- help="Start a Celery worker node",
- func=lazy_load_command("airflow.cli.commands.celery_command.worker"),
- args=(
- ARG_QUEUES,
- ARG_CONCURRENCY,
- ARG_CELERY_HOSTNAME,
- ARG_PID,
- ARG_DAEMON,
- ARG_UMASK,
- ARG_STDOUT,
- ARG_STDERR,
- ARG_LOG_FILE,
- ARG_AUTOSCALE,
- ARG_SKIP_SERVE_LOGS,
- ARG_WITHOUT_MINGLE,
- ARG_WITHOUT_GOSSIP,
- ARG_VERBOSE,
- ),
- ),
- ActionCommand(
- name="flower",
- help="Start a Celery Flower",
- func=lazy_load_command("airflow.cli.commands.celery_command.flower"),
- args=(
- ARG_FLOWER_HOSTNAME,
- ARG_FLOWER_PORT,
- ARG_FLOWER_CONF,
- ARG_FLOWER_URL_PREFIX,
- ARG_FLOWER_BASIC_AUTH,
- ARG_BROKER_API,
- ARG_PID,
- ARG_DAEMON,
- ARG_STDOUT,
- ARG_STDERR,
- ARG_LOG_FILE,
- ARG_VERBOSE,
- ),
- ),
- ActionCommand(
- name="stop",
- help="Stop the Celery worker gracefully",
-
func=lazy_load_command("airflow.cli.commands.celery_command.stop_worker"),
- args=(ARG_PID, ARG_VERBOSE),
- ),
-)
-
CONFIG_COMMANDS = (
ActionCommand(
name="get-value",
@@ -2109,9 +1970,6 @@ core_commands: list[CLICommand] = [
help="Manage DAGs",
subcommands=DAGS_COMMANDS,
),
- GroupCommand(
- name="kubernetes", help="Tools to help run the KubernetesExecutor",
subcommands=KUBERNETES_COMMANDS
- ),
GroupCommand(
name="tasks",
help="Manage tasks",
@@ -2298,15 +2156,6 @@ core_commands: list[CLICommand] = [
func=lazy_load_command("airflow.cli.commands.plugins_command.dump_plugins"),
args=(ARG_OUTPUT, ARG_VERBOSE),
),
- GroupCommand(
- name="celery",
- help="Celery components",
- description=(
- "Start celery components. Works only when using CeleryExecutor.
For more information, see "
-
"https://airflow.apache.org/docs/apache-airflow/stable/executor/celery.html"
- ),
- subcommands=CELERY_COMMANDS,
- ),
ActionCommand(
name="standalone",
help="Run an all-in-one copy of Airflow",
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
index 9ccccd0f30..1a8ec1b844 100644
--- a/airflow/cli/cli_parser.py
+++ b/airflow/cli/cli_parser.py
@@ -24,6 +24,7 @@
from __future__ import annotations
import argparse
+import logging
from argparse import Action
from functools import lru_cache
from typing import Iterable
@@ -41,10 +42,27 @@ from airflow.cli.cli_config import (
core_commands,
)
from airflow.exceptions import AirflowException
+from airflow.executors.executor_loader import ExecutorLoader
from airflow.utils.helpers import partition
airflow_commands = core_commands
+log = logging.getLogger(__name__)
+try:
+ executor, _ = ExecutorLoader.import_default_executor_cls(validate=False)
+ airflow_commands.extend(executor.get_cli_commands())
+except Exception:
+ executor_name = ExecutorLoader.get_default_executor_name()
+ log.exception("Failed to load CLI commands from executor: %s",
executor_name)
+ log.error(
+ "Ensure all dependencies are met and try again. If using a Celery
based executor install "
+ "a 3.3.0+ version of the Celery provider. If using a Kubernetes
executor, install a "
+ "7.4.0+ version of the CNCF provider"
+ )
+ # Do no re-raise the exception since we want the CLI to still function for
+ # other commands.
+
+
ALL_COMMANDS_DICT: dict[str, CLICommand] = {sp.name: sp for sp in
airflow_commands}
diff --git a/airflow/executors/base_executor.py
b/airflow/executors/base_executor.py
index 1a44540b07..999125afe7 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -27,6 +27,7 @@ from typing import TYPE_CHECKING, Any, List, Optional,
Sequence, Tuple
import pendulum
+from airflow.cli.cli_config import GroupCommand
from airflow.configuration import conf
from airflow.exceptions import RemovedInAirflow3Warning
from airflow.stats import Stats
@@ -479,3 +480,12 @@ class BaseExecutor(LoggingMixin):
if not self.callback_sink:
raise ValueError("Callback sink is not ready.")
self.callback_sink.send(request)
+
+ @staticmethod
+ def get_cli_commands() -> list[GroupCommand]:
+ """Vends CLI commands to be included in Airflow CLI.
+
+ Override this method to expose commands via Airflow CLI to manage this
executor. This can
+ be commands to setup/teardown the executor, inspect state, etc.
+ """
+ return []
diff --git a/airflow/executors/executor_loader.py
b/airflow/executors/executor_loader.py
index ca21bdf05a..4a4bda8583 100644
--- a/airflow/executors/executor_loader.py
+++ b/airflow/executors/executor_loader.py
@@ -119,18 +119,24 @@ class ExecutorLoader:
return executor_cls()
@classmethod
- def import_executor_cls(cls, executor_name: str) ->
tuple[type[BaseExecutor], ConnectorSource]:
+ def import_executor_cls(
+ cls, executor_name: str, validate: bool = True
+ ) -> tuple[type[BaseExecutor], ConnectorSource]:
"""
Imports the executor class.
Supports the same formats as ExecutorLoader.load_executor.
+ :param executor_name: Name of core executor or module path to provider
provided as a plugin.
+ :param validate: Whether or not to validate the executor before
returning
+
:return: executor class via executor_name and executor import source
"""
def _import_and_validate(path: str) -> type[BaseExecutor]:
executor = import_string(path)
- cls.validate_database_executor_compatibility(executor)
+ if validate:
+ cls.validate_database_executor_compatibility(executor)
return executor
if executor_name in cls.executors:
@@ -151,14 +157,16 @@ class ExecutorLoader:
return _import_and_validate(executor_name), ConnectorSource.CUSTOM_PATH
@classmethod
- def import_default_executor_cls(cls) -> tuple[type[BaseExecutor],
ConnectorSource]:
+ def import_default_executor_cls(cls, validate: bool = True) ->
tuple[type[BaseExecutor], ConnectorSource]:
"""
Imports the default executor class.
+ :param validate: Whether or not to validate the executor before
returning
+
:return: executor class and executor import source
"""
executor_name = cls.get_default_executor_name()
- executor, source = cls.import_executor_cls(executor_name)
+ executor, source = cls.import_executor_cls(executor_name,
validate=validate)
return executor, source
@classmethod
diff --git a/airflow/providers/celery/executors/celery_executor.py
b/airflow/providers/celery/executors/celery_executor.py
index d587be44db..10513605ba 100644
--- a/airflow/providers/celery/executors/celery_executor.py
+++ b/airflow/providers/celery/executors/celery_executor.py
@@ -34,6 +34,40 @@ from typing import TYPE_CHECKING, Any, Optional, Sequence,
Tuple
from celery import states as celery_states
+try:
+ from airflow.cli.cli_config import (
+ ARG_AUTOSCALE,
+ ARG_DAEMON,
+ ARG_LOG_FILE,
+ ARG_PID,
+ ARG_SKIP_SERVE_LOGS,
+ ARG_STDERR,
+ ARG_STDOUT,
+ ARG_VERBOSE,
+ ActionCommand,
+ Arg,
+ GroupCommand,
+ lazy_load_command,
+ )
+except ImportError:
+ try:
+ from airflow import __version__ as airflow_version
+ except ImportError:
+ from airflow.version import version as airflow_version
+
+ import packaging.version
+
+ from airflow.exceptions import AirflowOptionalProviderFeatureException
+
+ base_version = packaging.version.parse(airflow_version).base_version
+
+ if packaging.version.parse(base_version) <
packaging.version.parse("2.7.0"):
+ raise AirflowOptionalProviderFeatureException(
+ "Celery Executor from Celery Provider should only be used with
Airflow 2.7.0+.\n"
+ f"This is Airflow {airflow_version} and Celery and
CeleryKubernetesExecutor are "
+ f"available in the 'airflow.executors' package. You should not use
"
+ f"the provider's executors in this version of Airflow."
+ )
from airflow.configuration import conf
from airflow.exceptions import AirflowTaskTimeout
from airflow.executors.base_executor import BaseExecutor
@@ -76,6 +110,119 @@ airflow celery worker
"""
+# flower cli args
+ARG_BROKER_API = Arg(("-a", "--broker-api"), help="Broker API")
+ARG_FLOWER_HOSTNAME = Arg(
+ ("-H", "--hostname"),
+ default=conf.get("celery", "FLOWER_HOST"),
+ help="Set the hostname on which to run the server",
+)
+ARG_FLOWER_PORT = Arg(
+ ("-p", "--port"),
+ default=conf.getint("celery", "FLOWER_PORT"),
+ type=int,
+ help="The port on which to run the server",
+)
+ARG_FLOWER_CONF = Arg(("-c", "--flower-conf"), help="Configuration file for
flower")
+ARG_FLOWER_URL_PREFIX = Arg(
+ ("-u", "--url-prefix"),
+ default=conf.get("celery", "FLOWER_URL_PREFIX"),
+ help="URL prefix for Flower",
+)
+ARG_FLOWER_BASIC_AUTH = Arg(
+ ("-A", "--basic-auth"),
+ default=conf.get("celery", "FLOWER_BASIC_AUTH"),
+ help=(
+ "Securing Flower with Basic Authentication. "
+ "Accepts user:password pairs separated by a comma. "
+ "Example: flower_basic_auth = user1:password1,user2:password2"
+ ),
+)
+
+# worker cli args
+ARG_QUEUES = Arg(
+ ("-q", "--queues"),
+ help="Comma delimited list of queues to serve",
+ default=conf.get("operators", "DEFAULT_QUEUE"),
+)
+ARG_CONCURRENCY = Arg(
+ ("-c", "--concurrency"),
+ type=int,
+ help="The number of worker processes",
+ default=conf.getint("celery", "worker_concurrency"),
+)
+ARG_CELERY_HOSTNAME = Arg(
+ ("-H", "--celery-hostname"),
+ help="Set the hostname of celery worker if you have multiple workers on a
single machine",
+)
+ARG_UMASK = Arg(
+ ("-u", "--umask"),
+ help="Set the umask of celery worker in daemon mode",
+)
+
+ARG_WITHOUT_MINGLE = Arg(
+ ("--without-mingle",),
+ default=False,
+ help="Don't synchronize with other workers at start-up",
+ action="store_true",
+)
+ARG_WITHOUT_GOSSIP = Arg(
+ ("--without-gossip",),
+ default=False,
+ help="Don't subscribe to other workers events",
+ action="store_true",
+)
+
+CELERY_COMMANDS = (
+ ActionCommand(
+ name="worker",
+ help="Start a Celery worker node",
+ func=lazy_load_command("airflow.cli.commands.celery_command.worker"),
+ args=(
+ ARG_QUEUES,
+ ARG_CONCURRENCY,
+ ARG_CELERY_HOSTNAME,
+ ARG_PID,
+ ARG_DAEMON,
+ ARG_UMASK,
+ ARG_STDOUT,
+ ARG_STDERR,
+ ARG_LOG_FILE,
+ ARG_AUTOSCALE,
+ ARG_SKIP_SERVE_LOGS,
+ ARG_WITHOUT_MINGLE,
+ ARG_WITHOUT_GOSSIP,
+ ARG_VERBOSE,
+ ),
+ ),
+ ActionCommand(
+ name="flower",
+ help="Start a Celery Flower",
+ func=lazy_load_command("airflow.cli.commands.celery_command.flower"),
+ args=(
+ ARG_FLOWER_HOSTNAME,
+ ARG_FLOWER_PORT,
+ ARG_FLOWER_CONF,
+ ARG_FLOWER_URL_PREFIX,
+ ARG_FLOWER_BASIC_AUTH,
+ ARG_BROKER_API,
+ ARG_PID,
+ ARG_DAEMON,
+ ARG_STDOUT,
+ ARG_STDERR,
+ ARG_LOG_FILE,
+ ARG_VERBOSE,
+ ),
+ ),
+ ActionCommand(
+ name="stop",
+ help="Stop the Celery worker gracefully",
+
func=lazy_load_command("airflow.cli.commands.celery_command.stop_worker"),
+ args=(ARG_PID, ARG_VERBOSE),
+ ),
+)
+
+
class CeleryExecutor(BaseExecutor):
"""
CeleryExecutor is recommended for production use of Airflow.
@@ -317,3 +464,17 @@ class CeleryExecutor(BaseExecutor):
except Exception as ex:
self.log.error("Error revoking task instance %s from
celery: %s", task_instance_key, ex)
return readable_tis
+
+ @staticmethod
+ def get_cli_commands() -> list[GroupCommand]:
+ return [
+ GroupCommand(
+ name="celery",
+ help="Celery components",
+ description=(
+ "Start celery components. Works only when using
CeleryExecutor. For more information, "
+ "see
https://airflow.apache.org/docs/apache-airflow/stable/executor/celery.html"
+ ),
+ subcommands=CELERY_COMMANDS,
+ ),
+ ]
diff --git a/airflow/providers/celery/executors/celery_kubernetes_executor.py
b/airflow/providers/celery/executors/celery_kubernetes_executor.py
index c5bbaac081..725e7fe5c4 100644
--- a/airflow/providers/celery/executors/celery_kubernetes_executor.py
+++ b/airflow/providers/celery/executors/celery_kubernetes_executor.py
@@ -257,3 +257,7 @@ class CeleryKubernetesExecutor(LoggingMixin):
if not self.callback_sink:
raise ValueError("Callback sink is not ready.")
self.callback_sink.send(request)
+
+ @staticmethod
+ def get_cli_commands() -> list:
+ return CeleryExecutor.get_cli_commands() +
KubernetesExecutor.get_cli_commands()
diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
index e57ca205fe..1c4a0e91fb 100644
--- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
+++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
@@ -36,6 +36,39 @@ from typing import TYPE_CHECKING, Any, Sequence
from sqlalchemy.orm import Session
from airflow import AirflowException
+
+try:
+ from airflow.cli.cli_config import (
+ ARG_DAG_ID,
+ ARG_EXECUTION_DATE,
+ ARG_OUTPUT_PATH,
+ ARG_SUBDIR,
+ ARG_VERBOSE,
+ ActionCommand,
+ Arg,
+ GroupCommand,
+ lazy_load_command,
+ positive_int,
+ )
+except ImportError:
+ try:
+ from airflow import __version__ as airflow_version
+ except ImportError:
+ from airflow.version import version as airflow_version
+
+ import packaging.version
+
+ from airflow.exceptions import AirflowOptionalProviderFeatureException
+
+ base_version = packaging.version.parse(airflow_version).base_version
+
+ if packaging.version.parse(base_version) <
packaging.version.parse("2.7.0"):
+ raise AirflowOptionalProviderFeatureException(
+ "Kubernetes Executor from CNCF Provider should only be used with
Airflow 2.7.0+.\n"
+ f"This is Airflow {airflow_version} and Kubernetes and
CeleryKubernetesExecutor are "
+ f"available in the 'airflow.executors' package. You should not use
"
+ f"the provider's executors in this version of Airflow."
+ )
from airflow.configuration import conf
from airflow.executors.base_executor import BaseExecutor
from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types
import POD_EXECUTOR_DONE_KEY
@@ -70,6 +103,45 @@ class PodReconciliationError(AirflowException):
"""Raised when an error is encountered while trying to merge pod
configs."""
+# CLI Args
+ARG_NAMESPACE = Arg(
+ ("--namespace",),
+ default=conf.get("kubernetes_executor", "namespace"),
+ help="Kubernetes Namespace. Default value is `[kubernetes] namespace` in
configuration.",
+)
+
+ARG_MIN_PENDING_MINUTES = Arg(
+ ("--min-pending-minutes",),
+ default=30,
+ type=positive_int(allow_zero=False),
+ help=(
+ "Pending pods created before the time interval are to be cleaned up, "
+ "measured in minutes. Default value is 30(m). The minimum value is
5(m)."
+ ),
+)
+
+# CLI Commands
+KUBERNETES_COMMANDS = (
+ ActionCommand(
+ name="cleanup-pods",
+ help=(
+ "Clean up Kubernetes pods "
+ "(created by KubernetesExecutor/KubernetesPodOperator) "
+ "in evicted/failed/succeeded/pending states"
+ ),
+
func=lazy_load_command("airflow.cli.commands.kubernetes_command.cleanup_pods"),
+ args=(ARG_NAMESPACE, ARG_MIN_PENDING_MINUTES, ARG_VERBOSE),
+ ),
+ ActionCommand(
+ name="generate-dag-yaml",
+ help="Generate YAML files for all tasks in DAG. Useful for debugging
tasks without "
+ "launching into a cluster",
+
func=lazy_load_command("airflow.cli.commands.kubernetes_command.generate_pod_yaml"),
+ args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_SUBDIR, ARG_OUTPUT_PATH,
ARG_VERBOSE),
+ ),
+)
+
+
class KubernetesExecutor(BaseExecutor):
"""Executor for Kubernetes."""
@@ -644,3 +716,13 @@ class KubernetesExecutor(BaseExecutor):
def terminate(self):
"""Terminate the executor is not doing anything."""
+
+ @staticmethod
+ def get_cli_commands() -> list[GroupCommand]:
+ return [
+ GroupCommand(
+ name="kubernetes",
+ help="Tools to help run the KubernetesExecutor",
+ subcommands=KUBERNETES_COMMANDS,
+ )
+ ]
diff --git
a/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py
b/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py
index ee82252096..49977a35ef 100644
--- a/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py
+++ b/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py
@@ -240,3 +240,7 @@ class LocalKubernetesExecutor(LoggingMixin):
if not self.callback_sink:
raise ValueError("Callback sink is not ready.")
self.callback_sink.send(request)
+
+ @staticmethod
+ def get_cli_commands() -> list:
+ return KubernetesExecutor.get_cli_commands()
diff --git a/airflow/stats.py b/airflow/stats.py
index 2f45b338dc..5b84def6db 100644
--- a/airflow/stats.py
+++ b/airflow/stats.py
@@ -22,7 +22,6 @@ import socket
from typing import TYPE_CHECKING, Callable
from airflow.configuration import conf
-from airflow.metrics import datadog_logger, otel_logger, statsd_logger
from airflow.metrics.base_stats_logger import NoStatsLogger, StatsLogger
log = logging.getLogger(__name__)
@@ -46,10 +45,16 @@ class _Stats(type):
if not hasattr(cls.__class__, "factory"):
is_datadog_enabled_defined = conf.has_option("metrics",
"statsd_datadog_enabled")
if is_datadog_enabled_defined and conf.getboolean("metrics",
"statsd_datadog_enabled"):
+ from airflow.metrics import datadog_logger
+
cls.__class__.factory = datadog_logger.get_dogstatsd_logger
elif conf.getboolean("metrics", "statsd_on"):
+ from airflow.metrics import statsd_logger
+
cls.__class__.factory = statsd_logger.get_statsd_logger
elif conf.getboolean("metrics", "otel_on"):
+ from airflow.metrics import otel_logger
+
cls.__class__.factory = otel_logger.get_otel_logger
else:
cls.__class__.factory = NoStatsLogger
diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py
index 3069726a05..dcb8f5a9a7 100644
--- a/tests/cli/conftest.py
+++ b/tests/cli/conftest.py
@@ -34,6 +34,18 @@ custom_executor_module.CustomCeleryExecutor = type( # type:
ignore
custom_executor_module.CustomCeleryKubernetesExecutor = type( # type: ignore
"CustomCeleryKubernetesExecutor",
(celery_kubernetes_executor.CeleryKubernetesExecutor,), {}
)
+custom_executor_module.CustomCeleryExecutor = type( # type: ignore
+ "CustomLocalExecutor", (celery_executor.CeleryExecutor,), {}
+)
+custom_executor_module.CustomCeleryKubernetesExecutor = type( # type: ignore
+ "CustomLocalKubernetesExecutor",
(celery_kubernetes_executor.CeleryKubernetesExecutor,), {}
+)
+custom_executor_module.CustomCeleryExecutor = type( # type: ignore
+ "CustomKubernetesExecutor", (celery_executor.CeleryExecutor,), {}
+)
+custom_executor_module.CustomCeleryKubernetesExecutor = type( # type: ignore
+ "CustomCeleryKubernetesExecutor",
(celery_kubernetes_executor.CeleryKubernetesExecutor,), {}
+)
sys.modules["custom_executor"] = custom_executor_module
diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py
index 6089961de9..eaec31db99 100644
--- a/tests/cli/test_cli_parser.py
+++ b/tests/cli/test_cli_parser.py
@@ -27,6 +27,7 @@ import subprocess
import sys
import timeit
from collections import Counter
+from importlib import reload
from pathlib import Path
from unittest.mock import patch
@@ -203,31 +204,22 @@ class TestCli:
cli_config.positive_int(allow_zero=True)("-1")
@pytest.mark.parametrize(
- "command",
+ "executor",
[
- ["celery"],
- ["celery", "--help"],
- ["celery", "worker", "--help"],
- ["celery", "worker"],
- ["celery", "flower", "--help"],
- ["celery", "flower"],
- ["celery", "stop_worker", "--help"],
- ["celery", "stop_worker"],
+ "celery",
+ "kubernetes",
],
)
- def test_dag_parser_require_celery_executor(self, command):
+ def test_dag_parser_require_celery_executor(self, executor):
with conf_vars({("core", "executor"): "SequentialExecutor"}),
contextlib.redirect_stderr(
io.StringIO()
) as stderr:
+ reload(cli_parser)
parser = cli_parser.get_parser()
with pytest.raises(SystemExit):
- parser.parse_args(command)
+ parser.parse_args([executor])
stderr = stderr.getvalue()
- assert (
- "airflow command error: argument GROUP_OR_COMMAND: celery
subcommand "
- "works only with CeleryExecutor, CeleryKubernetesExecutor and
executors derived from them, "
- "your current executor: SequentialExecutor, subclassed from:
BaseExecutor, see help above."
- ) in stderr
+ assert (f"airflow command error: argument GROUP_OR_COMMAND: invalid
choice: '{executor}'") in stderr
@pytest.mark.parametrize(
"executor",
@@ -240,6 +232,7 @@ class TestCli:
)
def test_dag_parser_celery_command_accept_celery_executor(self, executor):
with conf_vars({("core", "executor"): executor}),
contextlib.redirect_stderr(io.StringIO()) as stderr:
+ reload(cli_parser)
parser = cli_parser.get_parser()
with pytest.raises(SystemExit):
parser.parse_args(["celery"])
@@ -248,6 +241,36 @@ class TestCli:
"airflow celery command error: the following arguments are
required: COMMAND, see help above."
) in stderr
+ @pytest.mark.parametrize(
+ "executor,expected_args",
+ [
+ ("CeleryExecutor", ["celery"]),
+ ("CeleryKubernetesExecutor", ["celery", "kubernetes"]),
+ ("custom_executor.CustomCeleryExecutor", ["celery"]),
+ ("custom_executor.CustomCeleryKubernetesExecutor", ["celery",
"kubernetes"]),
+ ("KubernetesExecutor", ["kubernetes"]),
+ ("custom_executor.KubernetesExecutor", ["kubernetes"]),
+ ("LocalExecutor", []),
+ ("LocalKubernetesExecutor", ["kubernetes"]),
+ ("custom_executor.LocalExecutor", []),
+ ("custom_executor.LocalKubernetesExecutor", ["kubernetes"]),
+ ("SequentialExecutor", []),
+ ],
+ )
+ def test_cli_parser_executors(self, executor, expected_args):
+ """Test that CLI commands for the configured executor are present"""
+ for expected_arg in expected_args:
+ with conf_vars({("core", "executor"): executor}),
contextlib.redirect_stderr(
+ io.StringIO()
+ ) as stderr:
+ reload(cli_parser)
+ parser = cli_parser.get_parser()
+ with pytest.raises(SystemExit) as e:
+ parser.parse_args([expected_arg, "--help"])
+ assert e.value.code == 0
+ stderr = stderr.getvalue()
+ assert "airflow command error" not in stderr
+
def test_dag_parser_config_command_dont_required_celery_executor(self):
with conf_vars({("core", "executor"): "CeleryExecutor"}),
contextlib.redirect_stderr(
io.StringIO()
diff --git a/tests/executors/test_base_executor.py
b/tests/executors/test_base_executor.py
index b2059ae996..7d7f21ac0d 100644
--- a/tests/executors/test_base_executor.py
+++ b/tests/executors/test_base_executor.py
@@ -62,6 +62,10 @@ def test_serve_logs_default_value():
assert not BaseExecutor.serve_logs
+def test_no_cli_commands_vended():
+ assert not BaseExecutor.get_cli_commands()
+
+
def test_get_event_buffer():
executor = BaseExecutor()
diff --git a/tests/integration/cli/commands/test_celery_command.py
b/tests/integration/cli/commands/test_celery_command.py
index b421c949f0..1b779272cc 100644
--- a/tests/integration/cli/commands/test_celery_command.py
+++ b/tests/integration/cli/commands/test_celery_command.py
@@ -17,6 +17,7 @@
from __future__ import annotations
+from importlib import reload
from unittest import mock
import pytest
@@ -31,7 +32,11 @@ from tests.test_utils.config import conf_vars
class TestWorkerServeLogs:
@classmethod
def setup_class(cls):
- cls.parser = cli_parser.get_parser()
+ with conf_vars({("core", "executor"): "CeleryExecutor"}):
+ # The cli_parser module is loaded during test collection. Reload
it here with the
+ # executor overridden so that we get the expected commands loaded.
+ reload(cli_parser)
+ cls.parser = cli_parser.get_parser()
@conf_vars({("core", "executor"): "CeleryExecutor"})
def test_serve_logs_on_worker_start(self):
diff --git a/tests/providers/celery/executors/test_celery_executor.py
b/tests/providers/celery/executors/test_celery_executor.py
index c8ddee8c4e..7390855f2e 100644
--- a/tests/providers/celery/executors/test_celery_executor.py
+++ b/tests/providers/celery/executors/test_celery_executor.py
@@ -111,6 +111,9 @@ class TestCeleryExecutor:
def test_supports_sentry(self):
assert CeleryExecutor.supports_sentry
+ def test_cli_commands_vended(self):
+ assert CeleryExecutor.get_cli_commands()
+
@pytest.mark.backend("mysql", "postgres")
def test_exception_propagation(self, caplog):
caplog.set_level(
diff --git
a/tests/providers/celery/executors/test_celery_kubernetes_executor.py
b/tests/providers/celery/executors/test_celery_kubernetes_executor.py
index 1950b93690..6c3857912b 100644
--- a/tests/providers/celery/executors/test_celery_kubernetes_executor.py
+++ b/tests/providers/celery/executors/test_celery_kubernetes_executor.py
@@ -49,6 +49,9 @@ class TestCeleryKubernetesExecutor:
def test_is_single_threaded_default_value(self):
assert not CeleryKubernetesExecutor.is_single_threaded
+ def test_cli_commands_vended(self):
+ assert CeleryKubernetesExecutor.get_cli_commands()
+
def test_queued_tasks(self):
celery_executor_mock = mock.MagicMock()
k8s_executor_mock = mock.MagicMock()
diff --git
a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
index 0b9721f117..f0d77d5c18 100644
--- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
+++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py
@@ -1192,6 +1192,9 @@ class TestKubernetesExecutor:
def test_supports_sentry(self):
assert not KubernetesExecutor.supports_sentry
+ def test_cli_commands_vended(self):
+ assert KubernetesExecutor.get_cli_commands()
+
def test_annotations_for_logging_task_metadata(self):
annotations_test = {
"dag_id": "dag",
diff --git
a/tests/providers/cncf/kubernetes/executors/test_local_kubernetes_executor.py
b/tests/providers/cncf/kubernetes/executors/test_local_kubernetes_executor.py
index ca40c95e45..2b1817ac89 100644
---
a/tests/providers/cncf/kubernetes/executors/test_local_kubernetes_executor.py
+++
b/tests/providers/cncf/kubernetes/executors/test_local_kubernetes_executor.py
@@ -46,6 +46,9 @@ class TestLocalKubernetesExecutor:
def test_is_single_threaded_default_value(self):
assert not LocalKubernetesExecutor.is_single_threaded
+ def test_cli_commands_vended(self):
+ assert LocalKubernetesExecutor.get_cli_commands()
+
def test_queued_tasks(self):
local_executor_mock = mock.MagicMock()
k8s_executor_mock = mock.MagicMock()