This is an automated email from the ASF dual-hosted git repository.

uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 2aa3cfb6ab Fix CLI commands when custom timetable is used (#32118)
2aa3cfb6ab is described below

commit 2aa3cfb6abd10779029b0c072493a1c1ed820b77
Author: Utkarsh Sharma <[email protected]>
AuthorDate: Mon Jul 10 13:37:45 2023 +0530

    Fix CLI commands when custom timetable is used (#32118)
    
    Co-authored-by: Tzu-ping Chung <[email protected]>
---
 airflow/cli/commands/dag_command.py     |  19 ++++--
 airflow/cli/commands/task_command.py    |   7 ++-
 airflow/models/dag.py                   |   5 ++
 tests/cli/commands/test_dag_command.py  |  38 ++++++++++++
 tests/cli/commands/test_task_command.py |  15 +++++
 tests/dags/test_workday_timetable.py    |  31 ++++++++++
 tests/plugins/test_plugins_manager.py   |   2 +-
 tests/plugins/workday.py                | 103 ++++++++++++++++++++++++++++++++
 8 files changed, 212 insertions(+), 8 deletions(-)

diff --git a/airflow/cli/commands/dag_command.py 
b/airflow/cli/commands/dag_command.py
index 9a76a1dffd..f57e26b38c 100644
--- a/airflow/cli/commands/dag_command.py
+++ b/airflow/cli/commands/dag_command.py
@@ -72,12 +72,19 @@ def _run_dag_backfill(dags: list[DAG], args) -> None:
 
         if args.dry_run:
             print(f"Dry run of DAG {dag.dag_id} on {args.start_date}")
-            dr = DagRun(dag.dag_id, execution_date=args.start_date)
-            for task in dag.tasks:
-                print(f"Task {task.task_id} located in DAG {dag.dag_id}")
-                ti = TaskInstance(task, run_id=None)
-                ti.dag_run = dr
-                ti.dry_run()
+            dagrun_infos = 
dag.iter_dagrun_infos_between(earliest=args.start_date, latest=args.end_date)
+            for dagrun_info in dagrun_infos:
+                dr = DagRun(
+                    dag.dag_id,
+                    execution_date=dagrun_info.logical_date,
+                    data_interval=dagrun_info.data_interval,
+                )
+
+                for task in dag.tasks:
+                    print(f"Task {task.task_id} located in DAG {dag.dag_id}")
+                    ti = TaskInstance(task, run_id=None)
+                    ti.dag_run = dr
+                    ti.dry_run()
         else:
             if args.reset_dagruns:
                 DAG.clear_dags(
diff --git a/airflow/cli/commands/task_command.py 
b/airflow/cli/commands/task_command.py
index 560764536b..a5e20e8751 100644
--- a/airflow/cli/commands/task_command.py
+++ b/airflow/cli/commands/task_command.py
@@ -131,7 +131,12 @@ def _get_dag_run(
         dag_run_execution_date = pendulum.instance(timezone.utcnow())
 
     if create_if_necessary == "memory":
-        dag_run = DagRun(dag.dag_id, run_id=exec_date_or_run_id, 
execution_date=dag_run_execution_date)
+        dag_run = DagRun(
+            dag.dag_id,
+            run_id=exec_date_or_run_id,
+            execution_date=dag_run_execution_date,
+            
data_interval=dag.timetable.infer_manual_data_interval(run_after=dag_run_execution_date),
+        )
         return dag_run, True
     elif create_if_necessary == "db":
         dag_run = dag.create_dagrun(
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index bd8e023f7a..c4ccb84bf0 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -2722,6 +2722,8 @@ class DAG(LoggingMixin):
             session=session,
         )
         self.log.debug("Getting dagrun for dag %s", self.dag_id)
+        logical_date = timezone.coerce_datetime(execution_date)
+        data_interval = 
self.timetable.infer_manual_data_interval(run_after=logical_date)
         dr: DagRun = _get_or_create_dagrun(
             dag=self,
             start_date=execution_date,
@@ -2729,6 +2731,7 @@ class DAG(LoggingMixin):
             run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date),
             session=session,
             conf=run_conf,
+            data_interval=data_interval,
         )
 
         tasks = self.task_dict
@@ -3895,6 +3898,7 @@ def _get_or_create_dagrun(
     execution_date: datetime,
     run_id: str,
     session: Session,
+    data_interval: tuple[datetime, datetime] | None = None,
 ) -> DagRun:
     """Create a DAG run, replacing an existing instance if needed to prevent 
collisions.
 
@@ -3922,6 +3926,7 @@ def _get_or_create_dagrun(
         start_date=start_date or execution_date,
         session=session,
         conf=conf,
+        data_interval=data_interval,
     )
     log.info("created dagrun %s", dr)
     return dr
diff --git a/tests/cli/commands/test_dag_command.py 
b/tests/cli/commands/test_dag_command.py
index a54d26a993..f45e877840 100644
--- a/tests/cli/commands/test_dag_command.py
+++ b/tests/cli/commands/test_dag_command.py
@@ -384,6 +384,31 @@ class TestCliDags:
             disable_retry=False,
         )
 
+    @mock.patch("workday.AfterWorkdayTimetable")
+    @mock.patch("airflow.models.taskinstance.TaskInstance.dry_run")
+    @mock.patch("airflow.cli.commands.dag_command.DagRun")
+    def test_backfill_with_custom_timetable(self, mock_dagrun, mock_dry_run, 
mock_AfterWorkdayTimetable):
+        """
+        when calling `dags backfill` on dag with custom timetable, the DagRun 
object should be created with
+         data_intervals.
+        """
+        start_date = DEFAULT_DATE + timedelta(days=1)
+        end_date = start_date + timedelta(days=1)
+        cli_args = self.parser.parse_args(
+            [
+                "dags",
+                "backfill",
+                "example_workday_timetable",
+                "--start-date",
+                start_date.isoformat(),
+                "--end-date",
+                end_date.isoformat(),
+                "--dry-run",
+            ]
+        )
+        dag_command.dag_backfill(cli_args)
+        assert "data_interval" in mock_dagrun.call_args.kwargs
+
     def test_next_execution(self, tmp_path):
         dag_test_list = [
             ("future_schedule_daily", "timedelta(days=5)", "'0 0 * * *'", 
"True"),
@@ -778,3 +803,16 @@ class TestCliDags:
         )
         mock_render_dag.assert_has_calls([mock.call(mock_get_dag.return_value, 
tis=[])])
         assert "SOURCE" in output
+
+    @mock.patch("workday.AfterWorkdayTimetable")
+    @mock.patch("airflow.models.dag._get_or_create_dagrun")
+    def test_dag_test_with_custom_timetable(self, mock__get_or_create_dagrun, 
_):
+        """
+        when calling `dags test` on dag with custom timetable, the DagRun 
object should be created with
+         data_intervals.
+        """
+        cli_args = self.parser.parse_args(
+            ["dags", "test", "example_workday_timetable", 
DEFAULT_DATE.isoformat()]
+        )
+        dag_command.dag_test(cli_args)
+        assert "data_interval" in mock__get_or_create_dagrun.call_args.kwargs
diff --git a/tests/cli/commands/test_task_command.py 
b/tests/cli/commands/test_task_command.py
index 646d76f47c..7c2eb61d32 100644
--- a/tests/cli/commands/test_task_command.py
+++ b/tests/cli/commands/test_task_command.py
@@ -33,6 +33,7 @@ from unittest.mock import sentinel
 
 import pendulum
 import pytest
+import sqlalchemy.exc
 
 from airflow import DAG
 from airflow.cli import cli_parser
@@ -499,6 +500,20 @@ class TestCliTasks:
         assert 'echo "2022-01-01"' in output
         assert 'echo "2022-01-08"' in output
 
+    @mock.patch("sqlalchemy.orm.session.Session.query")
+    @mock.patch("airflow.cli.commands.task_command.DagRun")
+    def test_task_render_with_custom_timetable(self, mock_dagrun, mock_query):
+        """
+        when calling `tasks render` on dag with custom timetable, the DagRun 
object should be created with
+         data_intervals.
+        """
+        mock_query.side_effect = sqlalchemy.exc.NoResultFound
+
+        task_command.task_render(
+            self.parser.parse_args(["tasks", "render", 
"example_workday_timetable", "run_this", "2022-01-01"])
+        )
+        assert "data_interval" in mock_dagrun.call_args.kwargs
+
     def test_cli_run_when_pickle_and_dag_cli_method_selected(self):
         """
         tasks run should return an AirflowException when invalid pickle_id is 
passed
diff --git a/tests/dags/test_workday_timetable.py 
b/tests/dags/test_workday_timetable.py
new file mode 100644
index 0000000000..c37aa97cc9
--- /dev/null
+++ b/tests/dags/test_workday_timetable.py
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pendulum
+from workday import AfterWorkdayTimetable
+
+from airflow import DAG
+from airflow.operators.empty import EmptyOperator
+
+with DAG(
+    dag_id="example_workday_timetable",
+    start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+    schedule=AfterWorkdayTimetable(),
+    tags=["example", "timetable"],
+):
+    EmptyOperator(task_id="run_this")
diff --git a/tests/plugins/test_plugins_manager.py 
b/tests/plugins/test_plugins_manager.py
index 5c5c6c9ae9..019e2a69a8 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -198,7 +198,7 @@ class TestPluginsManager:
         with mock.patch("airflow.plugins_manager.plugins", []):
             plugins_manager.load_plugins_from_plugin_directory()
 
-            assert 5 == len(plugins_manager.plugins)
+            assert 6 == len(plugins_manager.plugins)
             for plugin in plugins_manager.plugins:
                 if "AirflowTestOnLoadPlugin" not in str(plugin):
                     continue
diff --git a/tests/plugins/workday.py b/tests/plugins/workday.py
new file mode 100644
index 0000000000..20363a69e7
--- /dev/null
+++ b/tests/plugins/workday.py
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Plugin to demonstrate timetable registration and accommodate example 
DAGs."""
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+
+# [START howto_timetable]
+from pendulum import UTC, Date, DateTime, Time
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction, 
Timetable
+
+log = logging.getLogger(__name__)
+try:
+    from pandas.tseries.holiday import USFederalHolidayCalendar
+
+    holiday_calendar = USFederalHolidayCalendar()
+except ImportError:
+    log.warning("Could not import pandas. Holidays will not be considered.")
+    holiday_calendar = None
+
+
+class AfterWorkdayTimetable(Timetable):
+    def get_next_workday(self, d: DateTime, incr=1) -> DateTime:
+        next_start = d
+        while True:
+            if next_start.weekday() in (5, 6):  # If next start is in the 
weekend go to next day
+                next_start = next_start + incr * timedelta(days=1)
+                continue
+            if holiday_calendar is not None:
+                holidays = holiday_calendar.holidays(start=next_start, 
end=next_start).to_pydatetime()
+                if next_start in holidays:  # If next start is a holiday go to 
next day
+                    next_start = next_start + incr * timedelta(days=1)
+                    continue
+            break
+        return next_start
+
+    # [START howto_timetable_infer_manual_data_interval]
+    def infer_manual_data_interval(self, run_after: DateTime) -> DataInterval:
+        start = DateTime.combine((run_after - timedelta(days=1)).date(), 
Time.min).replace(tzinfo=UTC)
+        # Skip backwards over weekends and holidays to find last run
+        start = self.get_next_workday(start, incr=-1)
+        return DataInterval(start=start, end=(start + timedelta(days=1)))
+
+    # [END howto_timetable_infer_manual_data_interval]
+
+    # [START howto_timetable_next_dagrun_info]
+    def next_dagrun_info(
+        self,
+        *,
+        last_automated_data_interval: DataInterval | None,
+        restriction: TimeRestriction,
+    ) -> DagRunInfo | None:
+        if last_automated_data_interval is not None:  # There was a previous 
run on the regular schedule.
+            last_start = last_automated_data_interval.start
+            next_start = DateTime.combine((last_start + 
timedelta(days=1)).date(), Time.min).replace(
+                tzinfo=UTC
+            )
+        else:  # This is the first ever run on the regular schedule.
+            next_start = restriction.earliest
+            if next_start is None:  # No start_date. Don't schedule.
+                return None
+            if not restriction.catchup:
+                # If the DAG has catchup=False, today is the earliest to 
consider.
+                next_start = max(next_start, DateTime.combine(Date.today(), 
Time.min).replace(tzinfo=UTC))
+            elif next_start.time() != Time.min:
+                # If earliest does not fall on midnight, skip to the next day.
+                next_start = DateTime.combine(next_start.date() + 
timedelta(days=1), Time.min).replace(
+                    tzinfo=UTC
+                )
+        # Skip weekends and holidays
+        next_start = self.get_next_workday(next_start)
+
+        if restriction.latest is not None and next_start > restriction.latest:
+            return None  # Over the DAG's scheduled end; don't schedule.
+        return DagRunInfo.interval(start=next_start, end=(next_start + 
timedelta(days=1)))
+
+    # [END howto_timetable_next_dagrun_info]
+
+
+class WorkdayTimetablePlugin(AirflowPlugin):
+    name = "workday_timetable_plugin"
+    timetables = [AfterWorkdayTimetable]
+
+
+# [END howto_timetable]

Reply via email to