bugraoz93 commented on code in PR #45300: URL: https://github.com/apache/airflow/pull/45300#discussion_r1901109149
########## airflow/cli/api/operations.py: ########## @@ -0,0 +1,675 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import datetime +import sys +from typing import TYPE_CHECKING, Any + +import httpx +import rich +import structlog + +from airflow.cli.api.datamodels._generated import ( + AssetAliasCollectionResponse, + AssetAliasResponse, + AssetCollectionResponse, + AssetResponse, + BackfillPostBody, + BackfillResponse, + Config, + ConnectionBody, + ConnectionBulkBody, + ConnectionCollectionResponse, + ConnectionResponse, + ConnectionTestResponse, + DAGDetailsResponse, + DAGResponse, + DAGRunCollectionResponse, + DAGRunResponse, + JobCollectionResponse, + PoolCollectionResponse, + PoolPatchBody, + PoolPostBody, + PoolPostBulkBody, + PoolResponse, + ProviderCollectionResponse, + TriggerDAGRunPostBody, + VariableBody, + VariableCollectionResponse, + VariableResponse, + VersionInfo, +) + +if TYPE_CHECKING: + from airflow.cli.api.client import Client + from airflow.utils.state import DagRunState + +log = structlog.get_logger(logger_name=__name__) + + +# Generic Server Response Error +class ServerResponseError(httpx.HTTPStatusError): + """Server response error (Generic).""" + + def __init__(self, message: str, *, request: httpx.Request, response: httpx.Response): + super().__init__(message, request=request, response=response) + + # def_ + + @classmethod + def from_response(cls, response: httpx.Response) -> ServerResponseError | None: + if response.status_code < 400: + return None + + if response.headers.get("content-type") != "application/json": + return None + + if 400 <= response.status_code < 500: + response.read() + return cls( + message=f"Client error message: {response.json()}", + request=response.request, + response=response, + ) + + msg = response.json() + + self = cls(message=msg, request=response.request, response=response) + return self + + +# Decorator to apply methods to all operations, this is initiated at __init_subclass__ on BaseOperations +SERVER_CONNECTION_REFUSED_ERROR: str = "API Server is not running, please contact your administrator." + + +def _check_flag_and_exit_if_server_response_error(func): + """Return decorator to check for ServerResponseError and exit if the server is not running.""" + + def _exit_if_server_response_error(response: Any | ServerResponseError): + if isinstance(response, ServerResponseError): + rich.print(f"[bold red]Error:[/bold red] {response.response.json()}") + sys.exit(1) + return response + + def wrapped(self, *args, **kwargs): + try: + if self.exit_in_error: + return _exit_if_server_response_error(response=func(self, *args, **kwargs)) + else: + return func(self, *args, **kwargs) + except httpx.ConnectError: + rich.print(f"[bold red]{SERVER_CONNECTION_REFUSED_ERROR}[/bold red]") + sys.exit(1) + + return wrapped + + +class BaseOperations: + """ + Base class for operations. + + This class is used to decorate all callable methods with a check for ServerResponseError. + Set exit_in_error false to not exit. + """ + + __slots__ = ("client", "response", "exit_in_error") + + def __init__(self, client: Client, response=None, exit_in_error: bool = True): + self.client = client + self.response = response + self.exit_in_error = exit_in_error + + def __init_subclass__(cls, **kwargs): + """Decorate all callable methods with a check for ServerResponseError and exit if the server is not running.""" + super().__init_subclass__(**kwargs) + for attr, value in cls.__dict__.items(): + if callable(value): + setattr(cls, attr, _check_flag_and_exit_if_server_response_error(value)) + + +# Operations +# TODO: Get all with limit and offset to overcome default 100 limit for all list operations Review Comment: Great point! Thanks for bringing it up! If the queries are returning huge results, we should also have pagination on the CLI level in that case if the client is handling this pagination. Indeed, within this context ([AIP-81](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-81+Enhanced+Security+in+CLI+via+Integration+of+API)), what we are trying is to lift and shift the current functionality via using REST API. That's why I have included the comment as let's first select all as the same as current behaviour. **Current Functionality:** **Connection Export:** https://github.com/apache/airflow/blob/f236ab14cce14dd49aeadc66bdb47b2e40ba8bb0/airflow/cli/commands/remote_commands/connection_command.py#L190-L191 **Variable Export:** https://github.com/apache/airflow/blob/f236ab14cce14dd49aeadc66bdb47b2e40ba8bb0/airflow/cli/commands/remote_commands/variable_command.py#L123-L131 We are aiming to release this AIP in v3.0. I am planning to propose another AIP for improvements and generify the approaches in the CLI. This is one of the good fits for that project. How about we first finish the integration and as the next iteration, we can make these improvements that allow us to scale more on the CLI for v3.1 or v3.2? If you are interested in contributing to the CLI, maybe you can check the READY column in this project. https://github.com/orgs/apache/projects/415. ########## airflow/cli/api/cli_api_client.py: ########## @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import contextlib +from functools import wraps +from typing import Callable, TypeVar, cast + +import httpx + +from airflow.cli.api.client import Client, Credentials +from airflow.typing_compat import ParamSpec + +PS = ParamSpec("PS") +RT = TypeVar("RT") + + [email protected] +def get_client(): + """Get CLI API client.""" + cli_api_client = None + try: + credentials = Credentials.load() + limits = httpx.Limits(max_keepalive_connections=1, max_connections=10) + cli_api_client = Client( + base_url=credentials.api_url, + limits=limits, + ) + yield cli_api_client + finally: + if cli_api_client: + cli_api_client.close() + + +def provide_cli_api_client(func: Callable[PS, RT]) -> Callable[PS, RT]: + """ + Provide a CLI API Client to the decorated function. + + If you want to reuse a CLI API Client or run the function as part of + API call, you pass it to the function, if not this wrapper + will create one and close it for you. + """ + + @wraps(func) + def wrapper(*args, **kwargs) -> RT: + with get_client() as cli_api_client: Review Comment: No one should pass the `cli_api_client` if it is not and for mocking purposes in `unit tests`. The Client should be generic and singleton for all CLI commands. Users can run a single command using the `airflow` CLI, which can only run one command at a time. You can run parallel commands with different methods using the `airflow` CLI in the terminal, but each command running is an individual process behind the scenes on the OS level. I think this is why we shouldn't consider having multiple clients within the CLI context. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
