This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2aa3cfb6ab Fix CLI commands when custom timetable is used (#32118)
2aa3cfb6ab is described below
commit 2aa3cfb6abd10779029b0c072493a1c1ed820b77
Author: Utkarsh Sharma <[email protected]>
AuthorDate: Mon Jul 10 13:37:45 2023 +0530
Fix CLI commands when custom timetable is used (#32118)
Co-authored-by: Tzu-ping Chung <[email protected]>
---
airflow/cli/commands/dag_command.py | 19 ++++--
airflow/cli/commands/task_command.py | 7 ++-
airflow/models/dag.py | 5 ++
tests/cli/commands/test_dag_command.py | 38 ++++++++++++
tests/cli/commands/test_task_command.py | 15 +++++
tests/dags/test_workday_timetable.py | 31 ++++++++++
tests/plugins/test_plugins_manager.py | 2 +-
tests/plugins/workday.py | 103 ++++++++++++++++++++++++++++++++
8 files changed, 212 insertions(+), 8 deletions(-)
diff --git a/airflow/cli/commands/dag_command.py
b/airflow/cli/commands/dag_command.py
index 9a76a1dffd..f57e26b38c 100644
--- a/airflow/cli/commands/dag_command.py
+++ b/airflow/cli/commands/dag_command.py
@@ -72,12 +72,19 @@ def _run_dag_backfill(dags: list[DAG], args) -> None:
if args.dry_run:
print(f"Dry run of DAG {dag.dag_id} on {args.start_date}")
- dr = DagRun(dag.dag_id, execution_date=args.start_date)
- for task in dag.tasks:
- print(f"Task {task.task_id} located in DAG {dag.dag_id}")
- ti = TaskInstance(task, run_id=None)
- ti.dag_run = dr
- ti.dry_run()
+ dagrun_infos =
dag.iter_dagrun_infos_between(earliest=args.start_date, latest=args.end_date)
+ for dagrun_info in dagrun_infos:
+ dr = DagRun(
+ dag.dag_id,
+ execution_date=dagrun_info.logical_date,
+ data_interval=dagrun_info.data_interval,
+ )
+
+ for task in dag.tasks:
+ print(f"Task {task.task_id} located in DAG {dag.dag_id}")
+ ti = TaskInstance(task, run_id=None)
+ ti.dag_run = dr
+ ti.dry_run()
else:
if args.reset_dagruns:
DAG.clear_dags(
diff --git a/airflow/cli/commands/task_command.py
b/airflow/cli/commands/task_command.py
index 560764536b..a5e20e8751 100644
--- a/airflow/cli/commands/task_command.py
+++ b/airflow/cli/commands/task_command.py
@@ -131,7 +131,12 @@ def _get_dag_run(
dag_run_execution_date = pendulum.instance(timezone.utcnow())
if create_if_necessary == "memory":
- dag_run = DagRun(dag.dag_id, run_id=exec_date_or_run_id,
execution_date=dag_run_execution_date)
+ dag_run = DagRun(
+ dag.dag_id,
+ run_id=exec_date_or_run_id,
+ execution_date=dag_run_execution_date,
+
data_interval=dag.timetable.infer_manual_data_interval(run_after=dag_run_execution_date),
+ )
return dag_run, True
elif create_if_necessary == "db":
dag_run = dag.create_dagrun(
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index bd8e023f7a..c4ccb84bf0 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -2722,6 +2722,8 @@ class DAG(LoggingMixin):
session=session,
)
self.log.debug("Getting dagrun for dag %s", self.dag_id)
+ logical_date = timezone.coerce_datetime(execution_date)
+ data_interval =
self.timetable.infer_manual_data_interval(run_after=logical_date)
dr: DagRun = _get_or_create_dagrun(
dag=self,
start_date=execution_date,
@@ -2729,6 +2731,7 @@ class DAG(LoggingMixin):
run_id=DagRun.generate_run_id(DagRunType.MANUAL, execution_date),
session=session,
conf=run_conf,
+ data_interval=data_interval,
)
tasks = self.task_dict
@@ -3895,6 +3898,7 @@ def _get_or_create_dagrun(
execution_date: datetime,
run_id: str,
session: Session,
+ data_interval: tuple[datetime, datetime] | None = None,
) -> DagRun:
"""Create a DAG run, replacing an existing instance if needed to prevent
collisions.
@@ -3922,6 +3926,7 @@ def _get_or_create_dagrun(
start_date=start_date or execution_date,
session=session,
conf=conf,
+ data_interval=data_interval,
)
log.info("created dagrun %s", dr)
return dr
diff --git a/tests/cli/commands/test_dag_command.py
b/tests/cli/commands/test_dag_command.py
index a54d26a993..f45e877840 100644
--- a/tests/cli/commands/test_dag_command.py
+++ b/tests/cli/commands/test_dag_command.py
@@ -384,6 +384,31 @@ class TestCliDags:
disable_retry=False,
)
+ @mock.patch("workday.AfterWorkdayTimetable")
+ @mock.patch("airflow.models.taskinstance.TaskInstance.dry_run")
+ @mock.patch("airflow.cli.commands.dag_command.DagRun")
+ def test_backfill_with_custom_timetable(self, mock_dagrun, mock_dry_run,
mock_AfterWorkdayTimetable):
+ """
+ when calling `dags backfill` on dag with custom timetable, the DagRun
object should be created with
+ data_intervals.
+ """
+ start_date = DEFAULT_DATE + timedelta(days=1)
+ end_date = start_date + timedelta(days=1)
+ cli_args = self.parser.parse_args(
+ [
+ "dags",
+ "backfill",
+ "example_workday_timetable",
+ "--start-date",
+ start_date.isoformat(),
+ "--end-date",
+ end_date.isoformat(),
+ "--dry-run",
+ ]
+ )
+ dag_command.dag_backfill(cli_args)
+ assert "data_interval" in mock_dagrun.call_args.kwargs
+
def test_next_execution(self, tmp_path):
dag_test_list = [
("future_schedule_daily", "timedelta(days=5)", "'0 0 * * *'",
"True"),
@@ -778,3 +803,16 @@ class TestCliDags:
)
mock_render_dag.assert_has_calls([mock.call(mock_get_dag.return_value,
tis=[])])
assert "SOURCE" in output
+
+ @mock.patch("workday.AfterWorkdayTimetable")
+ @mock.patch("airflow.models.dag._get_or_create_dagrun")
+ def test_dag_test_with_custom_timetable(self, mock__get_or_create_dagrun,
_):
+ """
+ when calling `dags test` on dag with custom timetable, the DagRun
object should be created with
+ data_intervals.
+ """
+ cli_args = self.parser.parse_args(
+ ["dags", "test", "example_workday_timetable",
DEFAULT_DATE.isoformat()]
+ )
+ dag_command.dag_test(cli_args)
+ assert "data_interval" in mock__get_or_create_dagrun.call_args.kwargs
diff --git a/tests/cli/commands/test_task_command.py
b/tests/cli/commands/test_task_command.py
index 646d76f47c..7c2eb61d32 100644
--- a/tests/cli/commands/test_task_command.py
+++ b/tests/cli/commands/test_task_command.py
@@ -33,6 +33,7 @@ from unittest.mock import sentinel
import pendulum
import pytest
+import sqlalchemy.exc
from airflow import DAG
from airflow.cli import cli_parser
@@ -499,6 +500,20 @@ class TestCliTasks:
assert 'echo "2022-01-01"' in output
assert 'echo "2022-01-08"' in output
+ @mock.patch("sqlalchemy.orm.session.Session.query")
+ @mock.patch("airflow.cli.commands.task_command.DagRun")
+ def test_task_render_with_custom_timetable(self, mock_dagrun, mock_query):
+ """
+ when calling `tasks render` on dag with custom timetable, the DagRun
object should be created with
+ data_intervals.
+ """
+ mock_query.side_effect = sqlalchemy.exc.NoResultFound
+
+ task_command.task_render(
+ self.parser.parse_args(["tasks", "render",
"example_workday_timetable", "run_this", "2022-01-01"])
+ )
+ assert "data_interval" in mock_dagrun.call_args.kwargs
+
def test_cli_run_when_pickle_and_dag_cli_method_selected(self):
"""
tasks run should return an AirflowException when invalid pickle_id is
passed
diff --git a/tests/dags/test_workday_timetable.py
b/tests/dags/test_workday_timetable.py
new file mode 100644
index 0000000000..c37aa97cc9
--- /dev/null
+++ b/tests/dags/test_workday_timetable.py
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pendulum
+from workday import AfterWorkdayTimetable
+
+from airflow import DAG
+from airflow.operators.empty import EmptyOperator
+
+with DAG(
+ dag_id="example_workday_timetable",
+ start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
+ schedule=AfterWorkdayTimetable(),
+ tags=["example", "timetable"],
+):
+ EmptyOperator(task_id="run_this")
diff --git a/tests/plugins/test_plugins_manager.py
b/tests/plugins/test_plugins_manager.py
index 5c5c6c9ae9..019e2a69a8 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -198,7 +198,7 @@ class TestPluginsManager:
with mock.patch("airflow.plugins_manager.plugins", []):
plugins_manager.load_plugins_from_plugin_directory()
- assert 5 == len(plugins_manager.plugins)
+ assert 6 == len(plugins_manager.plugins)
for plugin in plugins_manager.plugins:
if "AirflowTestOnLoadPlugin" not in str(plugin):
continue
diff --git a/tests/plugins/workday.py b/tests/plugins/workday.py
new file mode 100644
index 0000000000..20363a69e7
--- /dev/null
+++ b/tests/plugins/workday.py
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Plugin to demonstrate timetable registration and accommodate example
DAGs."""
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+
+# [START howto_timetable]
+from pendulum import UTC, Date, DateTime, Time
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction,
Timetable
+
+log = logging.getLogger(__name__)
+try:
+ from pandas.tseries.holiday import USFederalHolidayCalendar
+
+ holiday_calendar = USFederalHolidayCalendar()
+except ImportError:
+ log.warning("Could not import pandas. Holidays will not be considered.")
+ holiday_calendar = None
+
+
+class AfterWorkdayTimetable(Timetable):
+ def get_next_workday(self, d: DateTime, incr=1) -> DateTime:
+ next_start = d
+ while True:
+ if next_start.weekday() in (5, 6): # If next start is in the
weekend go to next day
+ next_start = next_start + incr * timedelta(days=1)
+ continue
+ if holiday_calendar is not None:
+ holidays = holiday_calendar.holidays(start=next_start,
end=next_start).to_pydatetime()
+ if next_start in holidays: # If next start is a holiday go to
next day
+ next_start = next_start + incr * timedelta(days=1)
+ continue
+ break
+ return next_start
+
+ # [START howto_timetable_infer_manual_data_interval]
+ def infer_manual_data_interval(self, run_after: DateTime) -> DataInterval:
+ start = DateTime.combine((run_after - timedelta(days=1)).date(),
Time.min).replace(tzinfo=UTC)
+ # Skip backwards over weekends and holidays to find last run
+ start = self.get_next_workday(start, incr=-1)
+ return DataInterval(start=start, end=(start + timedelta(days=1)))
+
+ # [END howto_timetable_infer_manual_data_interval]
+
+ # [START howto_timetable_next_dagrun_info]
+ def next_dagrun_info(
+ self,
+ *,
+ last_automated_data_interval: DataInterval | None,
+ restriction: TimeRestriction,
+ ) -> DagRunInfo | None:
+ if last_automated_data_interval is not None: # There was a previous
run on the regular schedule.
+ last_start = last_automated_data_interval.start
+ next_start = DateTime.combine((last_start +
timedelta(days=1)).date(), Time.min).replace(
+ tzinfo=UTC
+ )
+ else: # This is the first ever run on the regular schedule.
+ next_start = restriction.earliest
+ if next_start is None: # No start_date. Don't schedule.
+ return None
+ if not restriction.catchup:
+ # If the DAG has catchup=False, today is the earliest to
consider.
+ next_start = max(next_start, DateTime.combine(Date.today(),
Time.min).replace(tzinfo=UTC))
+ elif next_start.time() != Time.min:
+ # If earliest does not fall on midnight, skip to the next day.
+ next_start = DateTime.combine(next_start.date() +
timedelta(days=1), Time.min).replace(
+ tzinfo=UTC
+ )
+ # Skip weekends and holidays
+ next_start = self.get_next_workday(next_start)
+
+ if restriction.latest is not None and next_start > restriction.latest:
+ return None # Over the DAG's scheduled end; don't schedule.
+ return DagRunInfo.interval(start=next_start, end=(next_start +
timedelta(days=1)))
+
+ # [END howto_timetable_next_dagrun_info]
+
+
+class WorkdayTimetablePlugin(AirflowPlugin):
+ name = "workday_timetable_plugin"
+ timetables = [AfterWorkdayTimetable]
+
+
+# [END howto_timetable]