This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1cb2c2df337 Remove `subarg` argument from `dag` commands and add
support for `bundle-name` argument (#46849)
1cb2c2df337 is described below
commit 1cb2c2df33725e68e62aa5b5e9b7d75fd2843096
Author: ambikagarg <[email protected]>
AuthorDate: Fri Mar 14 11:23:39 2025 -0400
Remove `subarg` argument from `dag` commands and add support for
`bundle-name` argument (#46849)
Co-authored-by: Jed Cunningham <[email protected]>
---
airflow/cli/cli_config.py | 19 ++---
.../cli/commands/remote_commands/dag_command.py | 82 +++++++++++++++---
newsfragments/aip-66.significant.rst | 10 ++-
.../commands/remote_commands/test_dag_command.py | 96 +++++++++++++---------
4 files changed, 142 insertions(+), 65 deletions(-)
diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py
index 754a8e8394d..4ab8da207ba 100644
--- a/airflow/cli/cli_config.py
+++ b/airflow/cli/cli_config.py
@@ -942,13 +942,13 @@ DAGS_COMMANDS = (
name="list-import-errors",
help="List all the DAGs that have import errors",
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_list_import_errors"),
- args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
+ args=(ARG_BUNDLE_NAME, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name="report",
help="Show DagBag loading report",
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_report"),
- args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
+ args=(ARG_BUNDLE_NAME, ARG_OUTPUT, ARG_VERBOSE),
),
ActionCommand(
name="list-runs",
@@ -981,7 +981,7 @@ DAGS_COMMANDS = (
name="state",
help="Get the status of a dag run",
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_state"),
- args=(ARG_DAG_ID, ARG_LOGICAL_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_VERBOSE),
+ args=(ARG_DAG_ID, ARG_LOGICAL_DATE_OR_RUN_ID, ARG_VERBOSE),
),
ActionCommand(
name="next-execution",
@@ -991,7 +991,7 @@ DAGS_COMMANDS = (
"num-executions option is given"
),
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_next_execution"),
- args=(ARG_DAG_ID, ARG_SUBDIR, ARG_NUM_EXECUTIONS, ARG_VERBOSE),
+ args=(ARG_DAG_ID, ARG_NUM_EXECUTIONS, ARG_VERBOSE),
),
ActionCommand(
name="pause",
@@ -1024,7 +1024,6 @@ DAGS_COMMANDS = (
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_trigger"),
args=(
ARG_DAG_ID,
- ARG_SUBDIR,
ARG_RUN_ID,
ARG_CONF,
ARG_EXEC_DATE,
@@ -1060,13 +1059,7 @@ DAGS_COMMANDS = (
"airflow dags show <DAG_ID> --save output.dot\n"
),
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_show"),
- args=(
- ARG_DAG_ID,
- ARG_SUBDIR,
- ARG_SAVE,
- ARG_IMGCAT,
- ARG_VERBOSE,
- ),
+ args=(ARG_DAG_ID, ARG_SAVE, ARG_IMGCAT, ARG_VERBOSE),
),
ActionCommand(
name="show-dependencies",
@@ -1090,7 +1083,6 @@ DAGS_COMMANDS = (
),
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_dependencies_show"),
args=(
- ARG_SUBDIR,
ARG_SAVE,
ARG_IMGCAT,
ARG_VERBOSE,
@@ -1123,7 +1115,6 @@ DAGS_COMMANDS = (
ARG_DAG_ID,
ARG_LOGICAL_DATE_OPTIONAL,
ARG_CONF,
- ARG_SUBDIR,
ARG_SHOW_DAGRUN,
ARG_IMGCAT_DAGRUN,
ARG_SAVE_DAGRUN,
diff --git a/airflow/cli/commands/remote_commands/dag_command.py
b/airflow/cli/commands/remote_commands/dag_command.py
index c105255dc7e..2d038282000 100644
--- a/airflow/cli/commands/remote_commands/dag_command.py
+++ b/airflow/cli/commands/remote_commands/dag_command.py
@@ -24,9 +24,11 @@ import errno
import json
import logging
import operator
+import os
import re
import subprocess
import sys
+from pathlib import Path
from typing import TYPE_CHECKING
from itsdangerous import URLSafeSerializer
@@ -46,7 +48,7 @@ from airflow.models.errors import ParseImportError
from airflow.models.serialized_dag import SerializedDagModel
from airflow.sdk.definitions._internal.dag_parsing_context import
_airflow_parsing_context_manager
from airflow.utils import cli as cli_utils, timezone
-from airflow.utils.cli import get_dag, process_subdir,
suppress_logs_and_warning, validate_dag_bundle_arg
+from airflow.utils.cli import get_dag, suppress_logs_and_warning,
validate_dag_bundle_arg
from airflow.utils.dot_renderer import render_dag, render_dag_dependencies
from airflow.utils.helpers import ask_yesno
from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
@@ -243,7 +245,7 @@ def dag_dependencies_show(args) -> None:
@providers_configuration_loaded
def dag_show(args) -> None:
"""Display DAG or saves its graphic representation to the file."""
- dag = get_dag(args.subdir, args.dag_id)
+ dag = get_dag(subdir=None, dag_id=args.dag_id, from_db=True)
dot = render_dag(dag)
filename = args.save
imgcat = args.imgcat
@@ -353,7 +355,7 @@ def dag_next_execution(args) -> None:
>>> airflow dags next-execution tutorial
2018-08-31 10:38:00
"""
- dag = get_dag(args.subdir, args.dag_id)
+ dag = get_dag(subdir=None, dag_id=args.dag_id, from_db=True)
with create_session() as session:
last_parsed_dag: DagModel = session.scalars(
@@ -473,12 +475,27 @@ def dag_details(args, session: Session = NEW_SESSION):
@cli_utils.action_cli
@suppress_logs_and_warning
@providers_configuration_loaded
-def dag_list_import_errors(args) -> None:
+@provide_session
+def dag_list_import_errors(args, session: Session = NEW_SESSION) -> None:
"""Display dags with import errors on the command line."""
- dagbag = DagBag(process_subdir(args.subdir))
data = []
- for filename, errors in dagbag.import_errors.items():
- data.append({"filepath": filename, "error": errors})
+
+ # Get import errors from the DB
+ query = select(ParseImportError)
+ if args.bundle_name:
+ validate_dag_bundle_arg(args.bundle_name)
+ query = query.where(ParseImportError.bundle_name.in_(args.bundle_name))
+
+ dagbag_import_errors = session.scalars(query).all()
+
+ for import_error in dagbag_import_errors:
+ data.append(
+ {
+ "bundle_name": import_error.bundle_name,
+ "filepath": import_error.filename,
+ "error": import_error.stacktrace,
+ }
+ )
AirflowConsole().print_as(
data=data,
output=args.output,
@@ -492,9 +509,25 @@ def dag_list_import_errors(args) -> None:
@providers_configuration_loaded
def dag_report(args) -> None:
"""Display dagbag stats at the command line."""
- dagbag = DagBag(process_subdir(args.subdir))
+ manager = DagBundlesManager()
+
+ all_bundles = list(manager.get_all_dag_bundles())
+ if args.bundle_name:
+ validate_dag_bundle_arg(args.bundle_name)
+ bundles_to_reserialize = set(args.bundle_name)
+ else:
+ bundles_to_reserialize = {b.name for b in all_bundles}
+
+ all_dagbag_stats = []
+ for bundle in all_bundles:
+ if bundle.name not in bundles_to_reserialize:
+ continue
+ bundle.initialize()
+ dagbag = DagBag(bundle.path, include_examples=False)
+ all_dagbag_stats.extend(dagbag.dagbag_stats)
+
AirflowConsole().print_as(
- data=dagbag.dagbag_stats,
+ data=all_dagbag_stats,
output=args.output,
mapper=lambda x: {
"file": x.file,
@@ -576,6 +609,29 @@ def dag_list_dag_runs(args, dag: DAG | None = None,
session: Session = NEW_SESSI
AirflowConsole().print_as(data=dag_runs, output=args.output,
mapper=_render_dagrun)
+def _parse_and_get_dag(dag_id: str) -> DAG | None:
+ """Given a dag_id, determine the bundle and relative fileloc from the db,
then parse and return the DAG."""
+ db_dag = get_dag(subdir=None, dag_id=dag_id, from_db=True)
+ bundle_name = db_dag.get_bundle_name()
+ if bundle_name is None:
+ raise AirflowException(
+ f"Bundle name for DAG {dag_id!r} is not found in the database.
This should not happen."
+ )
+ if db_dag.relative_fileloc is None:
+ raise AirflowException(
+ f"Relative fileloc for DAG {dag_id!r} is not found in the
database. This should not happen."
+ )
+ bundle = DagBundlesManager().get_bundle(bundle_name)
+ bundle.initialize()
+ dag_absolute_path = os.fspath(Path(bundle.path, db_dag.relative_fileloc))
+
+ with _airflow_parsing_context_manager(dag_id=dag_id):
+ bag = DagBag(
+ dag_folder=dag_absolute_path, include_examples=False,
safe_mode=False, load_op_links=False
+ )
+ return bag.dags.get(dag_id)
+
+
@cli_utils.action_cli
@providers_configuration_loaded
@provide_session
@@ -594,8 +650,12 @@ def dag_test(args, dag: DAG | None = None, session:
Session = NEW_SESSION) -> No
re.compile(args.mark_success_pattern) if args.mark_success_pattern is
not None else None
)
- with _airflow_parsing_context_manager(dag_id=args.dag_id):
- dag = dag or get_dag(subdir=args.subdir, dag_id=args.dag_id)
+ dag = dag or _parse_and_get_dag(args.dag_id)
+ if not dag:
+ raise AirflowException(
+ f"Dag {args.dag_id!r} could not be found; either it does not exist
or it failed to parse."
+ )
+
dr: DagRun = dag.test(
logical_date=logical_date,
run_conf=run_conf,
diff --git a/newsfragments/aip-66.significant.rst
b/newsfragments/aip-66.significant.rst
index deed154ca74..cd672192393 100644
--- a/newsfragments/aip-66.significant.rst
+++ b/newsfragments/aip-66.significant.rst
@@ -17,10 +17,18 @@ instead of their location on disk.
The ``-subdir`` option of the following commands has been replaced with
``--bundle-name``:
* ``airflow dag-processor``
+* ``airflow dags list-import-errors``
+* ``airflow dags report``
-The ``--subdir`` option has been removed from the following commands (it was a
noop):
+The ``--subdir`` option has been removed from the following commands:
+* ``airflow dags next-execution``
* ``airflow dags pause``
+* ``airflow dags show``
+* ``airflow dags show-dependencies``
+* ``airflow dags state``
+* ``airflow dags test``
+* ``airflow dags trigger``
* ``airflow dags unpause``
.. Provide additional contextual information
diff --git a/tests/cli/commands/remote_commands/test_dag_command.py
b/tests/cli/commands/remote_commands/test_dag_command.py
index 32b674826cc..411868d3889 100644
--- a/tests/cli/commands/remote_commands/test_dag_command.py
+++ b/tests/cli/commands/remote_commands/test_dag_command.py
@@ -20,6 +20,7 @@ from __future__ import annotations
import argparse
import contextlib
import json
+import logging
import os
from datetime import datetime, timedelta
from io import StringIO
@@ -200,16 +201,14 @@ class TestCliDags:
for dag_id in expected_output:
# Test num-executions = 1 (default)
- args = self.parser.parse_args(["dags", "next-execution", dag_id,
"-S", str(tmp_path)])
+ args = self.parser.parse_args(["dags", "next-execution", dag_id])
with contextlib.redirect_stdout(StringIO()) as temp_stdout:
dag_command.dag_next_execution(args)
out = temp_stdout.getvalue()
assert expected_output[dag_id][0] in out
# Test num-executions = 2
- args = self.parser.parse_args(
- ["dags", "next-execution", dag_id, "--num-executions", "2",
"-S", str(tmp_path)]
- )
+ args = self.parser.parse_args(["dags", "next-execution", dag_id,
"--num-executions", "2"])
with contextlib.redirect_stdout(StringIO()) as temp_stdout:
dag_command.dag_next_execution(args)
out = temp_stdout.getvalue()
@@ -316,18 +315,24 @@ class TestCliDags:
assert list(dag_details.keys()) == valid_cols
@conf_vars({("core", "load_examples"): "false"})
- def test_cli_list_import_errors(self):
- dag_path = os.path.join(TEST_DAGS_FOLDER, "test_invalid_cron.py")
+ def test_cli_list_import_errors(self, get_test_dag,
configure_testing_dag_bundle, caplog):
+ path_to_parse = TEST_DAGS_FOLDER / "test_invalid_cron.py"
+ get_test_dag("test_invalid_cron")
+
args = self.parser.parse_args(
- ["dags", "list-import-errors", "--output", "yaml", "--subdir",
dag_path]
+ ["dags", "list-import-errors", "--output", "yaml",
"--bundle-name", "testing"]
)
- with contextlib.redirect_stdout(StringIO()) as temp_stdout:
+
+ with configure_testing_dag_bundle(path_to_parse):
with pytest.raises(SystemExit) as err_ctx:
- dag_command.dag_list_import_errors(args)
- out = temp_stdout.getvalue()
- assert "[0 100 * * *] is not acceptable, out of range" in out
- assert dag_path in out
+ with caplog.at_level(logging.ERROR):
+ dag_command.dag_list_import_errors(args)
+
+ log_output = caplog.text
+
assert err_ctx.value.code == 1
+ assert str(path_to_parse) in log_output
+ assert "[0 100 * * *] is not acceptable, out of range" in log_output
def test_cli_list_dag_runs(self):
dag_command.dag_trigger(
@@ -560,37 +565,38 @@ class TestCliDags:
is None
)
- @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
- def test_dag_test(self, mock_get_dag):
+
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+ def test_dag_test(self, mock_parse_and_get_dag):
cli_args = self.parser.parse_args(["dags", "test",
"example_bash_operator", DEFAULT_DATE.isoformat()])
dag_command.dag_test(cli_args)
- mock_get_dag.assert_has_calls(
+ mock_parse_and_get_dag.assert_has_calls(
[
- mock.call(subdir=cli_args.subdir,
dag_id="example_bash_operator"),
+ mock.call("example_bash_operator"),
+ mock.call().__bool__(),
mock.call().test(
logical_date=timezone.parse(DEFAULT_DATE.isoformat()),
run_conf=None,
use_executor=False,
- session=mock.ANY,
mark_success_pattern=None,
+ session=mock.ANY,
),
]
)
- @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
- def test_dag_test_fail_raise_error(self, mock_get_dag):
+
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+ def test_dag_test_fail_raise_error(self, mock_parse_and_get_dag):
logical_date_str = DEFAULT_DATE.isoformat()
- mock_get_dag.return_value.test.return_value = DagRun(
+ mock_parse_and_get_dag.return_value.test.return_value = DagRun(
dag_id="example_bash_operator", logical_date=DEFAULT_DATE,
state=DagRunState.FAILED
)
cli_args = self.parser.parse_args(["dags", "test",
"example_bash_operator", logical_date_str])
with pytest.raises(SystemExit, match=r"DagRun failed"):
dag_command.dag_test(cli_args)
- @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
+
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
@mock.patch("airflow.utils.timezone.utcnow")
- def test_dag_test_no_logical_date(self, mock_utcnow, mock_get_dag):
+ def test_dag_test_no_logical_date(self, mock_utcnow,
mock_parse_and_get_dag):
now = pendulum.now()
mock_utcnow.return_value = now
cli_args = self.parser.parse_args(["dags", "test",
"example_bash_operator"])
@@ -599,9 +605,10 @@ class TestCliDags:
dag_command.dag_test(cli_args)
- mock_get_dag.assert_has_calls(
+ mock_parse_and_get_dag.assert_has_calls(
[
- mock.call(subdir=cli_args.subdir,
dag_id="example_bash_operator"),
+ mock.call("example_bash_operator"),
+ mock.call().__bool__(),
mock.call().test(
logical_date=mock.ANY,
run_conf=None,
@@ -612,8 +619,8 @@ class TestCliDags:
]
)
- @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
- def test_dag_test_conf(self, mock_get_dag):
+
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+ def test_dag_test_conf(self, mock_parse_and_get_dag):
cli_args = self.parser.parse_args(
[
"dags",
@@ -626,9 +633,10 @@ class TestCliDags:
)
dag_command.dag_test(cli_args)
- mock_get_dag.assert_has_calls(
+ mock_parse_and_get_dag.assert_has_calls(
[
- mock.call(subdir=cli_args.subdir,
dag_id="example_bash_operator"),
+ mock.call("example_bash_operator"),
+ mock.call().__bool__(),
mock.call().test(
logical_date=timezone.parse(DEFAULT_DATE.isoformat()),
run_conf={"dag_run_conf_param": "param_value"},
@@ -642,9 +650,11 @@ class TestCliDags:
@mock.patch(
"airflow.cli.commands.remote_commands.dag_command.render_dag",
return_value=MagicMock(source="SOURCE")
)
- @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
- def test_dag_test_show_dag(self, mock_get_dag, mock_render_dag):
- mock_get_dag.return_value.test.return_value.run_id =
"__test_dag_test_show_dag_fake_dag_run_run_id__"
+
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+ def test_dag_test_show_dag(self, mock_parse_and_get_dag, mock_render_dag):
+ mock_parse_and_get_dag.return_value.test.return_value.run_id = (
+ "__test_dag_test_show_dag_fake_dag_run_run_id__"
+ )
cli_args = self.parser.parse_args(
["dags", "test", "example_bash_operator",
DEFAULT_DATE.isoformat(), "--show-dagrun"]
@@ -654,9 +664,10 @@ class TestCliDags:
output = stdout.getvalue()
- mock_get_dag.assert_has_calls(
+ mock_parse_and_get_dag.assert_has_calls(
[
- mock.call(subdir=cli_args.subdir,
dag_id="example_bash_operator"),
+ mock.call("example_bash_operator"),
+ mock.call().__bool__(),
mock.call().test(
logical_date=timezone.parse(DEFAULT_DATE.isoformat()),
run_conf=None,
@@ -666,7 +677,7 @@ class TestCliDags:
),
]
)
- mock_render_dag.assert_has_calls([mock.call(mock_get_dag.return_value,
tis=[])])
+
mock_render_dag.assert_has_calls([mock.call(mock_parse_and_get_dag.return_value,
tis=[])])
assert "SOURCE" in output
@mock.patch("airflow.models.dag._get_or_create_dagrun")
@@ -685,14 +696,21 @@ class TestCliDags:
assert "data_interval" in mock__get_or_create_dagrun.call_args.kwargs
@mock.patch("airflow.models.dag._get_or_create_dagrun")
- def test_dag_with_parsing_context(self, mock__get_or_create_dagrun):
+ def test_dag_with_parsing_context(
+ self, mock__get_or_create_dagrun, testing_dag_bundle,
configure_testing_dag_bundle
+ ):
"""
airflow parsing context should be set when calling `dags test`.
"""
- cli_args = self.parser.parse_args(
- ["dags", "test", "test_dag_parsing_context",
DEFAULT_DATE.isoformat()]
- )
- dag_command.dag_test(cli_args)
+ path_to_parse = TEST_DAGS_FOLDER / "test_dag_parsing_context.py"
+
+ with configure_testing_dag_bundle(path_to_parse):
+ bag = DagBag(dag_folder=path_to_parse, include_examples=False)
+ bag.sync_to_db("testing", None)
+ cli_args = self.parser.parse_args(
+ ["dags", "test", "test_dag_parsing_context",
DEFAULT_DATE.isoformat()]
+ )
+ dag_command.dag_test(cli_args)
# if dag_parsing_context is not set, this DAG will only have 1 task
assert len(mock__get_or_create_dagrun.call_args[1]["dag"].task_ids) ==
2