pierrejeambrun commented on code in PR #43255:
URL: https://github.com/apache/airflow/pull/43255#discussion_r1816801659


##########
tests/api_fastapi/core_api/routes/public/test_dag_stats.py:
##########
@@ -0,0 +1,431 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+import pytest
+
+from airflow.models.dag import DagModel
+from airflow.models.dagrun import DagRun
+from airflow.utils import timezone
+from airflow.utils.state import DagRunState
+from airflow.utils.types import DagRunType
+
+from tests_common.test_utils.db import clear_db_dags, clear_db_runs, 
clear_db_serialized_dags
+
+pytestmark = pytest.mark.db_test
+
+DAG1_ID = "test_dag1"
+DAG2_ID = "test_dag2"
+DAG3_ID = "test_dag3"
+TASK_ID = "op1"
+API_PREFIX = "/public/dagStats"
+
+
+class TestDagStatsEndpoint:
+    default_time = "2020-06-11T18:00:00+00:00"
+
+    @staticmethod
+    def _clear_db():
+        clear_db_runs()
+        clear_db_dags()
+        clear_db_serialized_dags()
+
+    def _create_dag_and_runs(self, session=None):
+        dag_1 = DagModel(
+            dag_id=DAG1_ID,
+            fileloc="/tmp/dag_stats_1.py",
+            timetable_summary="2 2 * * *",
+            is_active=False,
+            is_paused=True,
+            owners="test_owner,another_test_owner",
+            next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+        )
+        dag_1_run_1 = DagRun(
+            dag_id=DAG1_ID,
+            run_id="test_dag_run_id_1",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="running",
+        )
+        dag_1_run_2 = DagRun(
+            dag_id=dag_1.dag_id,
+            run_id="test_dag_run_id_2",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time) + 
timedelta(days=1),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="failed",
+        )
+        dag_2 = DagModel(
+            dag_id=DAG2_ID,
+            fileloc="/tmp/dag_stats_2.py",
+            timetable_summary="2 2 * * *",
+            is_active=False,
+            is_paused=True,
+            owners="test_owner,another_test_owner",
+            next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+        )
+        dag_2_run_1 = DagRun(
+            dag_id=dag_2.dag_id,
+            run_id="test_dag_2_run_id_1",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="queued",
+        )
+        dag_3 = DagModel(
+            dag_id=DAG3_ID,
+            fileloc="/tmp/dag_stats_3.py",
+            timetable_summary="2 2 * * *",
+            is_active=False,
+            is_paused=True,
+            owners="test_owner,another_test_owner",
+            next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+        )
+        dag_3_run_1 = DagRun(
+            dag_id=dag_3.dag_id,
+            run_id="test_dag_3_run_id_1",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="success",
+        )
+        entities = (
+            dag_1,
+            dag_1_run_1,
+            dag_1_run_2,
+            dag_2,
+            dag_2_run_1,
+            dag_3,
+            dag_3_run_1,
+        )
+        for entity in entities:
+            session.add(entity)

Review Comment:
   ```suggestion
       session.addll(entities)
   ```



##########
tests/api_fastapi/core_api/routes/public/test_dag_stats.py:
##########
@@ -0,0 +1,431 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+import pytest
+
+from airflow.models.dag import DagModel
+from airflow.models.dagrun import DagRun
+from airflow.utils import timezone
+from airflow.utils.state import DagRunState
+from airflow.utils.types import DagRunType
+
+from tests_common.test_utils.db import clear_db_dags, clear_db_runs, 
clear_db_serialized_dags
+
+pytestmark = pytest.mark.db_test
+
+DAG1_ID = "test_dag1"
+DAG2_ID = "test_dag2"
+DAG3_ID = "test_dag3"
+TASK_ID = "op1"
+API_PREFIX = "/public/dagStats"
+
+
+class TestDagStatsEndpoint:
+    default_time = "2020-06-11T18:00:00+00:00"
+
+    @staticmethod
+    def _clear_db():
+        clear_db_runs()
+        clear_db_dags()
+        clear_db_serialized_dags()
+
+    def _create_dag_and_runs(self, session=None):
+        dag_1 = DagModel(
+            dag_id=DAG1_ID,
+            fileloc="/tmp/dag_stats_1.py",
+            timetable_summary="2 2 * * *",
+            is_active=False,
+            is_paused=True,
+            owners="test_owner,another_test_owner",
+            next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+        )
+        dag_1_run_1 = DagRun(
+            dag_id=DAG1_ID,
+            run_id="test_dag_run_id_1",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="running",
+        )
+        dag_1_run_2 = DagRun(
+            dag_id=dag_1.dag_id,
+            run_id="test_dag_run_id_2",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time) + 
timedelta(days=1),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="failed",
+        )
+        dag_2 = DagModel(
+            dag_id=DAG2_ID,
+            fileloc="/tmp/dag_stats_2.py",
+            timetable_summary="2 2 * * *",
+            is_active=False,
+            is_paused=True,
+            owners="test_owner,another_test_owner",
+            next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+        )
+        dag_2_run_1 = DagRun(
+            dag_id=dag_2.dag_id,
+            run_id="test_dag_2_run_id_1",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="queued",
+        )
+        dag_3 = DagModel(
+            dag_id=DAG3_ID,
+            fileloc="/tmp/dag_stats_3.py",
+            timetable_summary="2 2 * * *",
+            is_active=False,
+            is_paused=True,
+            owners="test_owner,another_test_owner",
+            next_dagrun=datetime(2021, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
+        )
+        dag_3_run_1 = DagRun(
+            dag_id=dag_3.dag_id,
+            run_id="test_dag_3_run_id_1",
+            run_type=DagRunType.MANUAL,
+            execution_date=timezone.parse(self.default_time),
+            start_date=timezone.parse(self.default_time),
+            external_trigger=True,
+            state="success",
+        )
+        entities = (
+            dag_1,
+            dag_1_run_1,
+            dag_1_run_2,
+            dag_2,
+            dag_2_run_1,
+            dag_3,
+            dag_3_run_1,
+        )
+        for entity in entities:
+            session.add(entity)
+        session.commit()
+
+    @pytest.fixture(autouse=True)
+    def setup(self) -> None:
+        self._clear_db()
+
+    def teardown_method(self) -> None:
+        self._clear_db()
+
+
+class TestGetDagStats(TestDagStatsEndpoint):
+    """Unit tests for Get DAG Stats."""
+
+    def test_should_respond_200(self, client, session):
+        self._create_dag_and_runs(session)
+        exp_payload = {
+            "dags": [
+                {
+                    "dag_id": DAG1_ID,
+                    "stats": [
+                        {
+                            "state": DagRunState.QUEUED,
+                            "count": 0,
+                        },
+                        {
+                            "state": DagRunState.RUNNING,
+                            "count": 1,
+                        },
+                        {
+                            "state": DagRunState.SUCCESS,
+                            "count": 0,
+                        },
+                        {
+                            "state": DagRunState.FAILED,
+                            "count": 1,
+                        },
+                    ],
+                },
+                {
+                    "dag_id": DAG2_ID,
+                    "stats": [
+                        {
+                            "state": DagRunState.QUEUED,
+                            "count": 1,
+                        },
+                        {
+                            "state": DagRunState.RUNNING,
+                            "count": 0,
+                        },
+                        {
+                            "state": DagRunState.SUCCESS,
+                            "count": 0,
+                        },
+                        {
+                            "state": DagRunState.FAILED,
+                            "count": 0,
+                        },
+                    ],
+                },
+            ],
+            "total_entries": 2,
+        }
+
+        response = client().get(f"{API_PREFIX}?dag_ids={DAG1_ID},{DAG2_ID}")
+        assert response.status_code == 200
+        res_json = response.json()
+        assert len(res_json["dags"]) == 2
+        assert sorted(
+            res_json["dags"],
+            key=lambda d: d["dag_id"],
+        ) == sorted(
+            exp_payload["dags"],
+            key=lambda d: d["dag_id"],
+        )

Review Comment:
   Why are we manually sorting the dag_id ? This should be enforced by the 
endpoint. A stable order should be returned, that we can test.



##########
airflow/api_fastapi/core_api/routes/public/dag_stats.py:
##########
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from fastapi import Depends
+from sqlalchemy import func, select
+from sqlalchemy.orm import Query, Session
+from typing_extensions import Annotated
+
+from airflow.api_fastapi.common.db.common import (
+    get_session,
+)
+from airflow.api_fastapi.common.router import AirflowRouter
+from airflow.api_fastapi.core_api.openapi.exceptions import 
create_openapi_http_exception_doc
+from airflow.api_fastapi.core_api.serializers.dag_stats import (
+    DagStatsCollectionResponse,
+)
+from airflow.models.dagrun import DagRun
+from airflow.utils.state import DagRunState
+
+dag_stats_router = AirflowRouter(tags=["DagStats"], prefix="/dagStats")
+
+
+@dag_stats_router.get(
+    "/",
+    responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422]),
+)
+async def get_dag_stats(
+    session: Annotated[Session, Depends(get_session)],
+    dag_ids: str | None = None,
+) -> DagStatsCollectionResponse:
+    """Get Dag statistics."""
+    query: Query = select(
+        DagRun.dag_id,
+        DagRun.state,
+        func.count(DagRun.state),
+    )
+    if dag_ids:
+        query_dag_ids = sorted(set(dag_ids.split(",")))

Review Comment:
   Not sure this `sorted` is of any use. The actual order is defined bellow by 
the `sorted(result_dag_id)`



##########
airflow/api_fastapi/core_api/routes/public/dag_stats.py:
##########
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from fastapi import Depends
+from sqlalchemy import func, select
+from sqlalchemy.orm import Query, Session
+from typing_extensions import Annotated
+
+from airflow.api_fastapi.common.db.common import (
+    get_session,
+)
+from airflow.api_fastapi.common.router import AirflowRouter
+from airflow.api_fastapi.core_api.openapi.exceptions import 
create_openapi_http_exception_doc
+from airflow.api_fastapi.core_api.serializers.dag_stats import (
+    DagStatsCollectionResponse,
+)
+from airflow.models.dagrun import DagRun
+from airflow.utils.state import DagRunState
+
+dag_stats_router = AirflowRouter(tags=["DagStats"], prefix="/dagStats")
+
+
+@dag_stats_router.get(
+    "/",
+    responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422]),
+)
+async def get_dag_stats(
+    session: Annotated[Session, Depends(get_session)],
+    dag_ids: str | None = None,
+) -> DagStatsCollectionResponse:
+    """Get Dag statistics."""
+    query: Query = select(
+        DagRun.dag_id,
+        DagRun.state,
+        func.count(DagRun.state),
+    )
+    if dag_ids:
+        query_dag_ids = sorted(set(dag_ids.split(",")))
+        query = query.where(DagRun.dag_id.in_(query_dag_ids))
+    query = query.group_by(DagRun.dag_id, DagRun.state)
+    query_result = session.execute(query)

Review Comment:
   Here you can add a default `order_by(Dag.dag_id)` in the query



##########
airflow/api_fastapi/core_api/routes/public/dag_stats.py:
##########
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from fastapi import Depends
+from sqlalchemy import func, select
+from sqlalchemy.orm import Query, Session
+from typing_extensions import Annotated
+
+from airflow.api_fastapi.common.db.common import (
+    get_session,
+)
+from airflow.api_fastapi.common.router import AirflowRouter
+from airflow.api_fastapi.core_api.openapi.exceptions import 
create_openapi_http_exception_doc
+from airflow.api_fastapi.core_api.serializers.dag_stats import (
+    DagStatsCollectionResponse,
+)
+from airflow.models.dagrun import DagRun
+from airflow.utils.state import DagRunState
+
+dag_stats_router = AirflowRouter(tags=["DagStats"], prefix="/dagStats")
+
+
+@dag_stats_router.get(
+    "/",
+    responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422]),
+)
+async def get_dag_stats(
+    session: Annotated[Session, Depends(get_session)],
+    dag_ids: str | None = None,
+) -> DagStatsCollectionResponse:
+    """Get Dag statistics."""
+    query: Query = select(
+        DagRun.dag_id,
+        DagRun.state,
+        func.count(DagRun.state),
+    )
+    if dag_ids:
+        query_dag_ids = sorted(set(dag_ids.split(",")))
+        query = query.where(DagRun.dag_id.in_(query_dag_ids))
+    query = query.group_by(DagRun.dag_id, DagRun.state)
+    query_result = session.execute(query)
+
+    result_dag_ids = set()

Review Comment:
   instead of using a set here, you can use a `list` and only `append` to it if 
it does not contain the `id`.
   
   This way you have `unique` `dag_ids` inside it, but the ordering is still 
maintained, and you response will have the appropriate order.



##########
airflow/api_fastapi/core_api/routes/public/dag_stats.py:
##########
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from fastapi import Depends
+from sqlalchemy import func, select
+from sqlalchemy.orm import Query, Session
+from typing_extensions import Annotated
+
+from airflow.api_fastapi.common.db.common import (
+    get_session,
+)
+from airflow.api_fastapi.common.router import AirflowRouter
+from airflow.api_fastapi.core_api.openapi.exceptions import 
create_openapi_http_exception_doc
+from airflow.api_fastapi.core_api.serializers.dag_stats import (
+    DagStatsCollectionResponse,
+    DagStatsResponse,
+    DagStatsStateResponse,
+)
+from airflow.models.dagrun import DagRun
+from airflow.utils.state import DagRunState
+
+dag_stats_router = AirflowRouter(tags=["DagStats"], prefix="/dagStats")
+
+
+@dag_stats_router.get(
+    "/",
+    responses=create_openapi_http_exception_doc([400, 401, 403, 404, 422]),
+)
+async def get_dag_stats(
+    session: Annotated[Session, Depends(get_session)],
+    dag_ids: str | None = None,

Review Comment:
   Yes, FastAPI handles list natively in the exploded way. This will be one of 
the API breaking change I believe and I don't think we want to manually handle 
list to make that backward compatible.
   
   HAving `?param=item1&param=item2` is the way to go with FastAPI.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to