minyeamer commented on code in PR #71558: URL: https://github.com/apache/airflow/pull/71558#discussion_r4192052323
########## airflow-core/src/airflow/api_fastapi/core_api/routes/ui/time_schedule.py: ########## @@ -0,0 +1,324 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from collections.abc import Generator +from datetime import timedelta +from itertools import groupby +from typing import Annotated, Literal + +import pendulum +from fastapi import Depends, HTTPException, Query, status +from fastapi.responses import StreamingResponse +from sqlalchemy import false, func, select +from sqlalchemy.sql import Select + +from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity +from airflow.api_fastapi.common.db.common import apply_filters_to_select +from airflow.api_fastapi.common.parameters import ( + FilterOptionEnum, + FilterParam, + QueryDagIdPatternSearch, + QueryDagRunRunTypesFilter, + QueryDagRunStateFilter, + QueryTagsFilter, + QueryTeamsFilter, + RangeFilter, + datetime_range_filter_factory, + filter_param_factory, + float_range_filter_factory, +) +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.datamodels.ui.time_schedule import TimeScheduleBatch, TimeScheduleItem +from airflow.api_fastapi.core_api.security import ReadableDagsFilterDep, requires_access_dag +from airflow.api_fastapi.core_api.services.ui.time_schedule import aggregate_time_schedule_items +from airflow.models import DagModel, DagRun +from airflow.models.serialized_dag import SerializedDagModel +from airflow.utils.session import create_session +from airflow.utils.state import DagRunState + +time_schedule_router = AirflowRouter(prefix="/time-schedule", tags=["Time Schedule"]) + +_DAG_BATCH_SIZE = 25 +_SUPPORTED_TIME_SCALES = (1, 5, 10, 15, 20, 30, 40, 50, 60) + + +def _build_run_item(dag_run: DagRun, *, label: str, is_time_scheduled: bool) -> TimeScheduleItem: + start_date = dag_run.start_date or dag_run.run_after + duration_ms = (dag_run.duration or 0) * 1000 + end_date = dag_run.end_date or start_date + timedelta(milliseconds=duration_ms) + return TimeScheduleItem( + dag_id=dag_run.dag_id, + dag_run_id=dag_run.run_id, + duration_ms=duration_ms, + end_date=end_date, + is_placeholder=False, + is_planned=False, + is_time_scheduled=is_time_scheduled, + label=label, + run_count=1, + start_date=start_date, + state=dag_run.state or DagRunState.QUEUED, + ) + + +def _build_selected_dags_query( + *, + dag_id_pattern: QueryDagIdPatternSearch, + readable_dags_filter: ReadableDagsFilterDep, + show_scheduled_only: bool, + tags: QueryTagsFilter, + teams: QueryTeamsFilter, + timetable_type: FilterParam[list[str] | None], +) -> Select[tuple[DagModel]]: + query = select(DagModel).where(DagModel.is_stale == false()) + query = apply_filters_to_select( + statement=query, + filters=[dag_id_pattern, tags, teams, timetable_type, readable_dags_filter], + ) + if show_scheduled_only: + query = query.where(DagModel.timetable_periodic.is_(True)) + return query + + +def _build_selected_runs_query( + *, + dag_ids_query: Select[tuple[DagModel]], + duration_range: RangeFilter, + limit: int, + run_after: RangeFilter, + run_type: QueryDagRunRunTypesFilter, + start_date_range: RangeFilter, + state: QueryDagRunStateFilter, +) -> Select[tuple[int, str]]: + query = select(DagRun.id, DagRun.dag_id).where( + DagRun.dag_id.in_(dag_ids_query.with_only_columns(DagModel.dag_id)) + ) + query = apply_filters_to_select( + statement=query, + filters=[run_after, start_date_range, duration_range, run_type, state], + ) + return query.order_by(func.coalesce(DagRun.start_date, DagRun.run_after).desc()).limit(limit) + + +@time_schedule_router.get( + "", + response_class=StreamingResponse, + response_model=TimeScheduleBatch, + dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.RUN))], + responses={ + 200: { + "content": {"application/x-ndjson": {"schema": {"type": "string"}}}, + "description": "NDJSON stream of aggregated Time Schedule batches", + } + }, +) +def get_time_schedule_stream( + tags: QueryTagsFilter, + teams: QueryTeamsFilter, + dag_id_pattern: QueryDagIdPatternSearch, + run_type: QueryDagRunRunTypesFilter, + state: QueryDagRunStateFilter, + readable_dags_filter: ReadableDagsFilterDep, + run_after: Annotated[RangeFilter, Depends(datetime_range_filter_factory("run_after", DagRun))], + start_date_range: Annotated[RangeFilter, Depends(datetime_range_filter_factory("start_date", DagRun))], + duration_range: Annotated[RangeFilter, Depends(float_range_filter_factory("duration", DagRun))], + timetable_type: Annotated[ + FilterParam[list[str] | None], + Depends(filter_param_factory(DagModel.timetable_type, list[str], FilterOptionEnum.IN)), + ], + aggregation_mode: Literal["max", "mean", "min"] = "mean", + limit: Annotated[int, Query(ge=1, le=5000)] = 200, + show_scheduled_only: bool = True, + time_scale: Annotated[ + int, + Query( + ge=min(_SUPPORTED_TIME_SCALES), + le=max(_SUPPORTED_TIME_SCALES), + json_schema_extra={"enum": _SUPPORTED_TIME_SCALES}, + ), + ] = 60, + timezone: str = "UTC", + view_mode: Literal["day", "week"] = "day", +) -> StreamingResponse: + """Stream filtered and aggregated Time Schedule data in Dag batches.""" + if time_scale not in _SUPPORTED_TIME_SCALES: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_CONTENT, + f"Unsupported time scale: {time_scale}", + ) + try: + pendulum.timezone(timezone) + except (ValueError, TypeError) as error: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_CONTENT, f"Unknown timezone: {timezone}") from error + + dag_ids_query = _build_selected_dags_query( + dag_id_pattern=dag_id_pattern, + readable_dags_filter=readable_dags_filter, + show_scheduled_only=show_scheduled_only, + tags=tags, + teams=teams, + timetable_type=timetable_type, + ) Review Comment: ```py from airflow.api_fastapi.common.parameters import ( ... QueryPausedFilter, ... ) def get_time_schedule_stream( ... paused: QueryPausedFilter, ``` ```suggestion dag_ids_query = _build_selected_dags_query( dag_id_pattern=dag_id_pattern, readable_dags_filter=readable_dags_filter, paused=paused, show_scheduled_only=show_scheduled_only, tags=tags, teams=teams, timetable_type=timetable_type, ) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
