This is an automated email from the ASF dual-hosted git repository.

jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 1cb2c2df337 Remove `subarg` argument from `dag` commands and add 
support for `bundle-name` argument (#46849)
1cb2c2df337 is described below

commit 1cb2c2df33725e68e62aa5b5e9b7d75fd2843096
Author: ambikagarg <[email protected]>
AuthorDate: Fri Mar 14 11:23:39 2025 -0400

    Remove `subarg` argument from `dag` commands and add support for 
`bundle-name` argument (#46849)
    
    Co-authored-by: Jed Cunningham <[email protected]>
---
 airflow/cli/cli_config.py                          | 19 ++---
 .../cli/commands/remote_commands/dag_command.py    | 82 +++++++++++++++---
 newsfragments/aip-66.significant.rst               | 10 ++-
 .../commands/remote_commands/test_dag_command.py   | 96 +++++++++++++---------
 4 files changed, 142 insertions(+), 65 deletions(-)

diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py
index 754a8e8394d..4ab8da207ba 100644
--- a/airflow/cli/cli_config.py
+++ b/airflow/cli/cli_config.py
@@ -942,13 +942,13 @@ DAGS_COMMANDS = (
         name="list-import-errors",
         help="List all the DAGs that have import errors",
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_list_import_errors"),
-        args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
+        args=(ARG_BUNDLE_NAME, ARG_OUTPUT, ARG_VERBOSE),
     ),
     ActionCommand(
         name="report",
         help="Show DagBag loading report",
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_report"),
-        args=(ARG_SUBDIR, ARG_OUTPUT, ARG_VERBOSE),
+        args=(ARG_BUNDLE_NAME, ARG_OUTPUT, ARG_VERBOSE),
     ),
     ActionCommand(
         name="list-runs",
@@ -981,7 +981,7 @@ DAGS_COMMANDS = (
         name="state",
         help="Get the status of a dag run",
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_state"),
-        args=(ARG_DAG_ID, ARG_LOGICAL_DATE_OR_RUN_ID, ARG_SUBDIR, ARG_VERBOSE),
+        args=(ARG_DAG_ID, ARG_LOGICAL_DATE_OR_RUN_ID, ARG_VERBOSE),
     ),
     ActionCommand(
         name="next-execution",
@@ -991,7 +991,7 @@ DAGS_COMMANDS = (
             "num-executions option is given"
         ),
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_next_execution"),
-        args=(ARG_DAG_ID, ARG_SUBDIR, ARG_NUM_EXECUTIONS, ARG_VERBOSE),
+        args=(ARG_DAG_ID, ARG_NUM_EXECUTIONS, ARG_VERBOSE),
     ),
     ActionCommand(
         name="pause",
@@ -1024,7 +1024,6 @@ DAGS_COMMANDS = (
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_trigger"),
         args=(
             ARG_DAG_ID,
-            ARG_SUBDIR,
             ARG_RUN_ID,
             ARG_CONF,
             ARG_EXEC_DATE,
@@ -1060,13 +1059,7 @@ DAGS_COMMANDS = (
             "airflow dags show <DAG_ID> --save output.dot\n"
         ),
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_show"),
-        args=(
-            ARG_DAG_ID,
-            ARG_SUBDIR,
-            ARG_SAVE,
-            ARG_IMGCAT,
-            ARG_VERBOSE,
-        ),
+        args=(ARG_DAG_ID, ARG_SAVE, ARG_IMGCAT, ARG_VERBOSE),
     ),
     ActionCommand(
         name="show-dependencies",
@@ -1090,7 +1083,6 @@ DAGS_COMMANDS = (
         ),
         
func=lazy_load_command("airflow.cli.commands.remote_commands.dag_command.dag_dependencies_show"),
         args=(
-            ARG_SUBDIR,
             ARG_SAVE,
             ARG_IMGCAT,
             ARG_VERBOSE,
@@ -1123,7 +1115,6 @@ DAGS_COMMANDS = (
             ARG_DAG_ID,
             ARG_LOGICAL_DATE_OPTIONAL,
             ARG_CONF,
-            ARG_SUBDIR,
             ARG_SHOW_DAGRUN,
             ARG_IMGCAT_DAGRUN,
             ARG_SAVE_DAGRUN,
diff --git a/airflow/cli/commands/remote_commands/dag_command.py 
b/airflow/cli/commands/remote_commands/dag_command.py
index c105255dc7e..2d038282000 100644
--- a/airflow/cli/commands/remote_commands/dag_command.py
+++ b/airflow/cli/commands/remote_commands/dag_command.py
@@ -24,9 +24,11 @@ import errno
 import json
 import logging
 import operator
+import os
 import re
 import subprocess
 import sys
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from itsdangerous import URLSafeSerializer
@@ -46,7 +48,7 @@ from airflow.models.errors import ParseImportError
 from airflow.models.serialized_dag import SerializedDagModel
 from airflow.sdk.definitions._internal.dag_parsing_context import 
_airflow_parsing_context_manager
 from airflow.utils import cli as cli_utils, timezone
-from airflow.utils.cli import get_dag, process_subdir, 
suppress_logs_and_warning, validate_dag_bundle_arg
+from airflow.utils.cli import get_dag, suppress_logs_and_warning, 
validate_dag_bundle_arg
 from airflow.utils.dot_renderer import render_dag, render_dag_dependencies
 from airflow.utils.helpers import ask_yesno
 from airflow.utils.providers_configuration_loader import 
providers_configuration_loaded
@@ -243,7 +245,7 @@ def dag_dependencies_show(args) -> None:
 @providers_configuration_loaded
 def dag_show(args) -> None:
     """Display DAG or saves its graphic representation to the file."""
-    dag = get_dag(args.subdir, args.dag_id)
+    dag = get_dag(subdir=None, dag_id=args.dag_id, from_db=True)
     dot = render_dag(dag)
     filename = args.save
     imgcat = args.imgcat
@@ -353,7 +355,7 @@ def dag_next_execution(args) -> None:
     >>> airflow dags next-execution tutorial
     2018-08-31 10:38:00
     """
-    dag = get_dag(args.subdir, args.dag_id)
+    dag = get_dag(subdir=None, dag_id=args.dag_id, from_db=True)
 
     with create_session() as session:
         last_parsed_dag: DagModel = session.scalars(
@@ -473,12 +475,27 @@ def dag_details(args, session: Session = NEW_SESSION):
 @cli_utils.action_cli
 @suppress_logs_and_warning
 @providers_configuration_loaded
-def dag_list_import_errors(args) -> None:
+@provide_session
+def dag_list_import_errors(args, session: Session = NEW_SESSION) -> None:
     """Display dags with import errors on the command line."""
-    dagbag = DagBag(process_subdir(args.subdir))
     data = []
-    for filename, errors in dagbag.import_errors.items():
-        data.append({"filepath": filename, "error": errors})
+
+    # Get import errors from the DB
+    query = select(ParseImportError)
+    if args.bundle_name:
+        validate_dag_bundle_arg(args.bundle_name)
+        query = query.where(ParseImportError.bundle_name.in_(args.bundle_name))
+
+    dagbag_import_errors = session.scalars(query).all()
+
+    for import_error in dagbag_import_errors:
+        data.append(
+            {
+                "bundle_name": import_error.bundle_name,
+                "filepath": import_error.filename,
+                "error": import_error.stacktrace,
+            }
+        )
     AirflowConsole().print_as(
         data=data,
         output=args.output,
@@ -492,9 +509,25 @@ def dag_list_import_errors(args) -> None:
 @providers_configuration_loaded
 def dag_report(args) -> None:
     """Display dagbag stats at the command line."""
-    dagbag = DagBag(process_subdir(args.subdir))
+    manager = DagBundlesManager()
+
+    all_bundles = list(manager.get_all_dag_bundles())
+    if args.bundle_name:
+        validate_dag_bundle_arg(args.bundle_name)
+        bundles_to_reserialize = set(args.bundle_name)
+    else:
+        bundles_to_reserialize = {b.name for b in all_bundles}
+
+    all_dagbag_stats = []
+    for bundle in all_bundles:
+        if bundle.name not in bundles_to_reserialize:
+            continue
+        bundle.initialize()
+        dagbag = DagBag(bundle.path, include_examples=False)
+        all_dagbag_stats.extend(dagbag.dagbag_stats)
+
     AirflowConsole().print_as(
-        data=dagbag.dagbag_stats,
+        data=all_dagbag_stats,
         output=args.output,
         mapper=lambda x: {
             "file": x.file,
@@ -576,6 +609,29 @@ def dag_list_dag_runs(args, dag: DAG | None = None, 
session: Session = NEW_SESSI
     AirflowConsole().print_as(data=dag_runs, output=args.output, 
mapper=_render_dagrun)
 
 
+def _parse_and_get_dag(dag_id: str) -> DAG | None:
+    """Given a dag_id, determine the bundle and relative fileloc from the db, 
then parse and return the DAG."""
+    db_dag = get_dag(subdir=None, dag_id=dag_id, from_db=True)
+    bundle_name = db_dag.get_bundle_name()
+    if bundle_name is None:
+        raise AirflowException(
+            f"Bundle name for DAG {dag_id!r} is not found in the database. 
This should not happen."
+        )
+    if db_dag.relative_fileloc is None:
+        raise AirflowException(
+            f"Relative fileloc for DAG {dag_id!r} is not found in the 
database. This should not happen."
+        )
+    bundle = DagBundlesManager().get_bundle(bundle_name)
+    bundle.initialize()
+    dag_absolute_path = os.fspath(Path(bundle.path, db_dag.relative_fileloc))
+
+    with _airflow_parsing_context_manager(dag_id=dag_id):
+        bag = DagBag(
+            dag_folder=dag_absolute_path, include_examples=False, 
safe_mode=False, load_op_links=False
+        )
+        return bag.dags.get(dag_id)
+
+
 @cli_utils.action_cli
 @providers_configuration_loaded
 @provide_session
@@ -594,8 +650,12 @@ def dag_test(args, dag: DAG | None = None, session: 
Session = NEW_SESSION) -> No
         re.compile(args.mark_success_pattern) if args.mark_success_pattern is 
not None else None
     )
 
-    with _airflow_parsing_context_manager(dag_id=args.dag_id):
-        dag = dag or get_dag(subdir=args.subdir, dag_id=args.dag_id)
+    dag = dag or _parse_and_get_dag(args.dag_id)
+    if not dag:
+        raise AirflowException(
+            f"Dag {args.dag_id!r} could not be found; either it does not exist 
or it failed to parse."
+        )
+
     dr: DagRun = dag.test(
         logical_date=logical_date,
         run_conf=run_conf,
diff --git a/newsfragments/aip-66.significant.rst 
b/newsfragments/aip-66.significant.rst
index deed154ca74..cd672192393 100644
--- a/newsfragments/aip-66.significant.rst
+++ b/newsfragments/aip-66.significant.rst
@@ -17,10 +17,18 @@ instead of their location on disk.
 The ``-subdir`` option of the following commands has been replaced with 
``--bundle-name``:
 
 * ``airflow dag-processor``
+* ``airflow dags list-import-errors``
+* ``airflow dags report``
 
-The ``--subdir`` option has been removed from the following commands (it was a 
noop):
+The ``--subdir`` option has been removed from the following commands:
 
+* ``airflow dags next-execution``
 * ``airflow dags pause``
+* ``airflow dags show``
+* ``airflow dags show-dependencies``
+* ``airflow dags state``
+* ``airflow dags test``
+* ``airflow dags trigger``
 * ``airflow dags unpause``
 
 .. Provide additional contextual information
diff --git a/tests/cli/commands/remote_commands/test_dag_command.py 
b/tests/cli/commands/remote_commands/test_dag_command.py
index 32b674826cc..411868d3889 100644
--- a/tests/cli/commands/remote_commands/test_dag_command.py
+++ b/tests/cli/commands/remote_commands/test_dag_command.py
@@ -20,6 +20,7 @@ from __future__ import annotations
 import argparse
 import contextlib
 import json
+import logging
 import os
 from datetime import datetime, timedelta
 from io import StringIO
@@ -200,16 +201,14 @@ class TestCliDags:
 
         for dag_id in expected_output:
             # Test num-executions = 1 (default)
-            args = self.parser.parse_args(["dags", "next-execution", dag_id, 
"-S", str(tmp_path)])
+            args = self.parser.parse_args(["dags", "next-execution", dag_id])
             with contextlib.redirect_stdout(StringIO()) as temp_stdout:
                 dag_command.dag_next_execution(args)
                 out = temp_stdout.getvalue()
             assert expected_output[dag_id][0] in out
 
             # Test num-executions = 2
-            args = self.parser.parse_args(
-                ["dags", "next-execution", dag_id, "--num-executions", "2", 
"-S", str(tmp_path)]
-            )
+            args = self.parser.parse_args(["dags", "next-execution", dag_id, 
"--num-executions", "2"])
             with contextlib.redirect_stdout(StringIO()) as temp_stdout:
                 dag_command.dag_next_execution(args)
                 out = temp_stdout.getvalue()
@@ -316,18 +315,24 @@ class TestCliDags:
         assert list(dag_details.keys()) == valid_cols
 
     @conf_vars({("core", "load_examples"): "false"})
-    def test_cli_list_import_errors(self):
-        dag_path = os.path.join(TEST_DAGS_FOLDER, "test_invalid_cron.py")
+    def test_cli_list_import_errors(self, get_test_dag, 
configure_testing_dag_bundle, caplog):
+        path_to_parse = TEST_DAGS_FOLDER / "test_invalid_cron.py"
+        get_test_dag("test_invalid_cron")
+
         args = self.parser.parse_args(
-            ["dags", "list-import-errors", "--output", "yaml", "--subdir", 
dag_path]
+            ["dags", "list-import-errors", "--output", "yaml", 
"--bundle-name", "testing"]
         )
-        with contextlib.redirect_stdout(StringIO()) as temp_stdout:
+
+        with configure_testing_dag_bundle(path_to_parse):
             with pytest.raises(SystemExit) as err_ctx:
-                dag_command.dag_list_import_errors(args)
-            out = temp_stdout.getvalue()
-        assert "[0 100 * * *] is not acceptable, out of range" in out
-        assert dag_path in out
+                with caplog.at_level(logging.ERROR):
+                    dag_command.dag_list_import_errors(args)
+
+        log_output = caplog.text
+
         assert err_ctx.value.code == 1
+        assert str(path_to_parse) in log_output
+        assert "[0 100 * * *] is not acceptable, out of range" in log_output
 
     def test_cli_list_dag_runs(self):
         dag_command.dag_trigger(
@@ -560,37 +565,38 @@ class TestCliDags:
             is None
         )
 
-    @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
-    def test_dag_test(self, mock_get_dag):
+    
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+    def test_dag_test(self, mock_parse_and_get_dag):
         cli_args = self.parser.parse_args(["dags", "test", 
"example_bash_operator", DEFAULT_DATE.isoformat()])
         dag_command.dag_test(cli_args)
 
-        mock_get_dag.assert_has_calls(
+        mock_parse_and_get_dag.assert_has_calls(
             [
-                mock.call(subdir=cli_args.subdir, 
dag_id="example_bash_operator"),
+                mock.call("example_bash_operator"),
+                mock.call().__bool__(),
                 mock.call().test(
                     logical_date=timezone.parse(DEFAULT_DATE.isoformat()),
                     run_conf=None,
                     use_executor=False,
-                    session=mock.ANY,
                     mark_success_pattern=None,
+                    session=mock.ANY,
                 ),
             ]
         )
 
-    @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
-    def test_dag_test_fail_raise_error(self, mock_get_dag):
+    
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+    def test_dag_test_fail_raise_error(self, mock_parse_and_get_dag):
         logical_date_str = DEFAULT_DATE.isoformat()
-        mock_get_dag.return_value.test.return_value = DagRun(
+        mock_parse_and_get_dag.return_value.test.return_value = DagRun(
             dag_id="example_bash_operator", logical_date=DEFAULT_DATE, 
state=DagRunState.FAILED
         )
         cli_args = self.parser.parse_args(["dags", "test", 
"example_bash_operator", logical_date_str])
         with pytest.raises(SystemExit, match=r"DagRun failed"):
             dag_command.dag_test(cli_args)
 
-    @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
+    
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
     @mock.patch("airflow.utils.timezone.utcnow")
-    def test_dag_test_no_logical_date(self, mock_utcnow, mock_get_dag):
+    def test_dag_test_no_logical_date(self, mock_utcnow, 
mock_parse_and_get_dag):
         now = pendulum.now()
         mock_utcnow.return_value = now
         cli_args = self.parser.parse_args(["dags", "test", 
"example_bash_operator"])
@@ -599,9 +605,10 @@ class TestCliDags:
 
         dag_command.dag_test(cli_args)
 
-        mock_get_dag.assert_has_calls(
+        mock_parse_and_get_dag.assert_has_calls(
             [
-                mock.call(subdir=cli_args.subdir, 
dag_id="example_bash_operator"),
+                mock.call("example_bash_operator"),
+                mock.call().__bool__(),
                 mock.call().test(
                     logical_date=mock.ANY,
                     run_conf=None,
@@ -612,8 +619,8 @@ class TestCliDags:
             ]
         )
 
-    @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
-    def test_dag_test_conf(self, mock_get_dag):
+    
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+    def test_dag_test_conf(self, mock_parse_and_get_dag):
         cli_args = self.parser.parse_args(
             [
                 "dags",
@@ -626,9 +633,10 @@ class TestCliDags:
         )
         dag_command.dag_test(cli_args)
 
-        mock_get_dag.assert_has_calls(
+        mock_parse_and_get_dag.assert_has_calls(
             [
-                mock.call(subdir=cli_args.subdir, 
dag_id="example_bash_operator"),
+                mock.call("example_bash_operator"),
+                mock.call().__bool__(),
                 mock.call().test(
                     logical_date=timezone.parse(DEFAULT_DATE.isoformat()),
                     run_conf={"dag_run_conf_param": "param_value"},
@@ -642,9 +650,11 @@ class TestCliDags:
     @mock.patch(
         "airflow.cli.commands.remote_commands.dag_command.render_dag", 
return_value=MagicMock(source="SOURCE")
     )
-    @mock.patch("airflow.cli.commands.remote_commands.dag_command.get_dag")
-    def test_dag_test_show_dag(self, mock_get_dag, mock_render_dag):
-        mock_get_dag.return_value.test.return_value.run_id = 
"__test_dag_test_show_dag_fake_dag_run_run_id__"
+    
@mock.patch("airflow.cli.commands.remote_commands.dag_command._parse_and_get_dag")
+    def test_dag_test_show_dag(self, mock_parse_and_get_dag, mock_render_dag):
+        mock_parse_and_get_dag.return_value.test.return_value.run_id = (
+            "__test_dag_test_show_dag_fake_dag_run_run_id__"
+        )
 
         cli_args = self.parser.parse_args(
             ["dags", "test", "example_bash_operator", 
DEFAULT_DATE.isoformat(), "--show-dagrun"]
@@ -654,9 +664,10 @@ class TestCliDags:
 
         output = stdout.getvalue()
 
-        mock_get_dag.assert_has_calls(
+        mock_parse_and_get_dag.assert_has_calls(
             [
-                mock.call(subdir=cli_args.subdir, 
dag_id="example_bash_operator"),
+                mock.call("example_bash_operator"),
+                mock.call().__bool__(),
                 mock.call().test(
                     logical_date=timezone.parse(DEFAULT_DATE.isoformat()),
                     run_conf=None,
@@ -666,7 +677,7 @@ class TestCliDags:
                 ),
             ]
         )
-        mock_render_dag.assert_has_calls([mock.call(mock_get_dag.return_value, 
tis=[])])
+        
mock_render_dag.assert_has_calls([mock.call(mock_parse_and_get_dag.return_value,
 tis=[])])
         assert "SOURCE" in output
 
     @mock.patch("airflow.models.dag._get_or_create_dagrun")
@@ -685,14 +696,21 @@ class TestCliDags:
         assert "data_interval" in mock__get_or_create_dagrun.call_args.kwargs
 
     @mock.patch("airflow.models.dag._get_or_create_dagrun")
-    def test_dag_with_parsing_context(self, mock__get_or_create_dagrun):
+    def test_dag_with_parsing_context(
+        self, mock__get_or_create_dagrun, testing_dag_bundle, 
configure_testing_dag_bundle
+    ):
         """
         airflow parsing context should be set when calling `dags test`.
         """
-        cli_args = self.parser.parse_args(
-            ["dags", "test", "test_dag_parsing_context", 
DEFAULT_DATE.isoformat()]
-        )
-        dag_command.dag_test(cli_args)
+        path_to_parse = TEST_DAGS_FOLDER / "test_dag_parsing_context.py"
+
+        with configure_testing_dag_bundle(path_to_parse):
+            bag = DagBag(dag_folder=path_to_parse, include_examples=False)
+            bag.sync_to_db("testing", None)
+            cli_args = self.parser.parse_args(
+                ["dags", "test", "test_dag_parsing_context", 
DEFAULT_DATE.isoformat()]
+            )
+            dag_command.dag_test(cli_args)
 
         # if dag_parsing_context is not set, this DAG will only have 1 task
         assert len(mock__get_or_create_dagrun.call_args[1]["dag"].task_ids) == 
2

Reply via email to