This is an automated email from the ASF dual-hosted git repository.
pierrejeambrun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7ab9b897b9 Restructure FastAPI API (#42128)
7ab9b897b9 is described below
commit 7ab9b897b99046395002859c45ab3a65e9fcae24
Author: Pierre Jeambrun <pierrejb...@gmail.com>
AuthorDate: Wed Sep 11 01:12:40 2024 +0800
Restructure FastAPI API (#42128)
* Restructure FastAPI API
* Fix CI
---
.pre-commit-config.yaml | 10 ++--
airflow/{api_ui => api_fastapi}/__init__.py | 0
airflow/{api_ui => api_fastapi}/app.py | 22 ++++---
airflow/{api_ui => api_fastapi}/gunicorn_config.py | 2 +-
airflow/{api_ui => api_fastapi}/main.py | 2 +-
.../openapi/v1-generated.yaml | 7 ++-
airflow/{api_ui => api_fastapi}/views/__init__.py | 0
.../views/public/__init__.py} | 4 +-
.../api_fastapi/views/ui/__init__.py | 10 ++--
.../views => api_fastapi/views/ui}/datasets.py | 2 +-
airflow/cli/cli_config.py | 40 ++++++-------
.../{ui_api_command.py => fastapi_api_command.py} | 24 ++++----
airflow/cli/commands/standalone_command.py | 8 +--
contributing-docs/03_contributors_quick_start.rst | 4 +-
contributing-docs/08_static_code_checks.rst | 2 +-
contributing-docs/14_node_environment_setup.rst | 4 +-
contributing-docs/testing/unit_tests.rst | 2 +-
dev/breeze/doc/03_developer_tasks.rst | 6 +-
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
.../src/airflow_breeze/params/shell_params.py | 4 +-
dev/breeze/src/airflow_breeze/utils/run_tests.py | 2 +-
.../src/airflow_breeze/utils/selective_checks.py | 4 +-
dev/breeze/src/airflow_breeze/utils/visuals.py | 6 +-
.../tests/test_pytest_args_for_test_types.py | 6 +-
dev/breeze/tests/test_selective_checks.py | 2 +-
docs/spelling_wordlist.txt | 1 +
scripts/ci/docker-compose/base-ports.yml | 2 +-
.../ci/pre_commit/check_tests_in_right_folders.py | 2 +-
...e_ui_api_spec.py => update_fastapi_api_spec.py} | 2 +-
scripts/in_container/bin/run_tmux | 4 +-
..._api_spec.py => run_update_fastapi_api_spec.py} | 12 +++-
tests/{api_ui => api_fastapi}/__init__.py | 0
tests/{api_ui => api_fastapi}/conftest.py | 2 +-
tests/{api_ui => api_fastapi}/views/__init__.py | 0
.../api_fastapi/views/ui}/__init__.py | 0
.../views/ui}/test_datasets.py | 0
..._api_command.py => test_fastapi_api_command.py} | 68 ++++++++++++----------
37 files changed, 138 insertions(+), 130 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 39fd712e33..6d6f77b343 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -135,7 +135,7 @@ repos:
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all YAML files except Helm templates
- exclude:
^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*|.*/reproducible_build.yaml$|^airflow/api_ui/openapi/v1-generated.yaml$
+ exclude:
^\.github/.*$|^.*/.*_vendor/|^chart/templates/.*|.*/reproducible_build.yaml$|^airflow/api_fastapi/openapi/v1-generated.yaml$
types: [yaml]
files: \.ya?ml$
args:
@@ -589,7 +589,7 @@ repos:
^airflow/api_connexion/openapi/v1.yaml$|
^airflow/ui/openapi-gen/|
^airflow/cli/commands/internal_api_command.py$|
- ^airflow/cli/commands/ui_api_command.py$|
+ ^airflow/cli/commands/fastapi_api_command.py$|
^airflow/cli/commands/webserver_command.py$|
^airflow/config_templates/|
^airflow/models/baseoperator.py$|
@@ -1330,11 +1330,11 @@ repos:
files:
^airflow/migrations/versions/.*\.py$|^docs/apache-airflow/migrations-ref\.rst$
additional_dependencies: ['rich>=12.4.4']
- id: generate-openapi-spec
- name: Generate the UI API OPENAPI spec
+ name: Generate the FastAPI API spec
language: python
- entry: ./scripts/ci/pre_commit/update_ui_api_spec.py
+ entry: ./scripts/ci/pre_commit/update_fastapi_api_spec.py
pass_filenames: false
- files: ^airflow/api_ui/.*\.py$
+ files: ^airflow/api_fastapi/.*\.py$
additional_dependencies: ['rich>=12.4.4']
- id: update-er-diagram
name: Update ER diagram
diff --git a/airflow/api_ui/__init__.py b/airflow/api_fastapi/__init__.py
similarity index 100%
copy from airflow/api_ui/__init__.py
copy to airflow/api_fastapi/__init__.py
diff --git a/airflow/api_ui/app.py b/airflow/api_fastapi/app.py
similarity index 73%
rename from airflow/api_ui/app.py
rename to airflow/api_fastapi/app.py
index b279f6c90d..f54c844461 100644
--- a/airflow/api_ui/app.py
+++ b/airflow/api_fastapi/app.py
@@ -16,7 +16,7 @@
# under the License.
from __future__ import annotations
-from fastapi import APIRouter, FastAPI
+from fastapi import FastAPI
from airflow.www.extensions.init_dagbag import get_dag_bag
@@ -34,9 +34,9 @@ def init_dag_bag(app: FastAPI) -> None:
def create_app() -> FastAPI:
app = FastAPI(
- description="Internal Rest API for the UI frontend. It is subject to
breaking change "
- "depending on the need of the frontend. Users should not rely on this
API but use the "
- "public API instead."
+ description="Airflow API. All endpoints located under ``/public`` can
be used safely, are stable and backward compatible. "
+ "Endpoints located under ``/ui`` are dedicated to the UI and are
subject to breaking change "
+ "depending on the need of the frontend. Users should not rely on those
but use the public ones instead."
)
init_dag_bag(app)
@@ -48,16 +48,14 @@ def create_app() -> FastAPI:
def init_views(app) -> None:
"""Init views by registering the different routers."""
- from airflow.api_ui.views.datasets import dataset_router
+ from airflow.api_fastapi.views.public import public_router
+ from airflow.api_fastapi.views.ui import ui_router
- root_router = APIRouter(prefix="/ui")
+ app.include_router(ui_router)
+ app.include_router(public_router)
- root_router.include_router(dataset_router)
- app.include_router(root_router)
-
-
-def cached_app(config=None, testing=False):
+def cached_app(config=None, testing=False) -> FastAPI:
"""Return cached instance of Airflow UI app."""
global app
if not app:
@@ -65,7 +63,7 @@ def cached_app(config=None, testing=False):
return app
-def purge_cached_app():
+def purge_cached_app() -> None:
"""Remove the cached version of the app in global state."""
global app
app = None
diff --git a/airflow/api_ui/gunicorn_config.py
b/airflow/api_fastapi/gunicorn_config.py
similarity index 91%
rename from airflow/api_ui/gunicorn_config.py
rename to airflow/api_fastapi/gunicorn_config.py
index 3ee3ba5e7c..70072ad6bd 100644
--- a/airflow/api_ui/gunicorn_config.py
+++ b/airflow/api_fastapi/gunicorn_config.py
@@ -27,7 +27,7 @@ def post_worker_init(_):
"""
Set process title.
- This is used by airflow.cli.commands.ui_api_command to track the status of
the worker.
+ This is used by airflow.cli.commands.fastapi_api_command to track the
status of the worker.
"""
old_title = setproctitle.getproctitle()
setproctitle.setproctitle(settings.GUNICORN_WORKER_READY_PREFIX +
old_title)
diff --git a/airflow/api_ui/main.py b/airflow/api_fastapi/main.py
similarity index 94%
copy from airflow/api_ui/main.py
copy to airflow/api_fastapi/main.py
index 175db6b927..9a204306ef 100644
--- a/airflow/api_ui/main.py
+++ b/airflow/api_fastapi/main.py
@@ -17,6 +17,6 @@
from __future__ import annotations
-from airflow.api_ui.app import cached_app
+from airflow.api_fastapi.app import cached_app
app = cached_app()
diff --git a/airflow/api_ui/openapi/v1-generated.yaml
b/airflow/api_fastapi/openapi/v1-generated.yaml
similarity index 81%
rename from airflow/api_ui/openapi/v1-generated.yaml
rename to airflow/api_fastapi/openapi/v1-generated.yaml
index fef897a071..dcd67b84df 100644
--- a/airflow/api_ui/openapi/v1-generated.yaml
+++ b/airflow/api_fastapi/openapi/v1-generated.yaml
@@ -1,9 +1,10 @@
openapi: 3.1.0
info:
title: FastAPI
- description: Internal Rest API for the UI frontend. It is subject to
breaking change
- depending on the need of the frontend. Users should not rely on this API
but use
- the public API instead.
+ description: Airflow API. All endpoints located under ``/public`` can be
used safely,
+ are stable and backward compatible. Endpoints located under ``/ui`` are
dedicated
+ to the UI and are subject to breaking change depending on the need of the
frontend.
+ Users should not rely on those but use the public ones instead.
version: 0.1.0
paths:
/ui/next_run_datasets/{dag_id}:
diff --git a/airflow/api_ui/views/__init__.py
b/airflow/api_fastapi/views/__init__.py
similarity index 100%
rename from airflow/api_ui/views/__init__.py
rename to airflow/api_fastapi/views/__init__.py
diff --git a/airflow/api_ui/main.py
b/airflow/api_fastapi/views/public/__init__.py
similarity index 91%
rename from airflow/api_ui/main.py
rename to airflow/api_fastapi/views/public/__init__.py
index 175db6b927..230b4a26c3 100644
--- a/airflow/api_ui/main.py
+++ b/airflow/api_fastapi/views/public/__init__.py
@@ -17,6 +17,6 @@
from __future__ import annotations
-from airflow.api_ui.app import cached_app
+from fastapi import APIRouter
-app = cached_app()
+public_router = APIRouter(prefix="/public")
diff --git a/tests/api_ui/conftest.py b/airflow/api_fastapi/views/ui/__init__.py
similarity index 82%
copy from tests/api_ui/conftest.py
copy to airflow/api_fastapi/views/ui/__init__.py
index 9f82802142..aa539e2845 100644
--- a/tests/api_ui/conftest.py
+++ b/airflow/api_fastapi/views/ui/__init__.py
@@ -16,12 +16,10 @@
# under the License.
from __future__ import annotations
-import pytest
-from fastapi.testclient import TestClient
+from fastapi import APIRouter
-from airflow.api_ui.app import create_app
+from airflow.api_fastapi.views.ui.datasets import dataset_router
+ui_router = APIRouter(prefix="/ui")
-@pytest.fixture
-def test_client():
- return TestClient(create_app())
+ui_router.include_router(dataset_router)
diff --git a/airflow/api_ui/views/datasets.py
b/airflow/api_fastapi/views/ui/datasets.py
similarity index 97%
rename from airflow/api_ui/views/datasets.py
rename to airflow/api_fastapi/views/ui/datasets.py
index 2ab983082f..d6de8ebca0 100644
--- a/airflow/api_ui/views/datasets.py
+++ b/airflow/api_fastapi/views/ui/datasets.py
@@ -29,7 +29,7 @@ dataset_router = APIRouter(tags=["Dataset"])
# Ultimately we want async routes, with async sqlalchemy session / context
manager.
# Additional effort to make airflow utility code async, not handled for now
and most likely part of the AIP-70
-@dataset_router.get("/next_run_datasets/{dag_id}")
+@dataset_router.get("/next_run_datasets/{dag_id}", include_in_schema=False)
async def next_run_datasets(dag_id: str, request: Request) -> dict:
dag = request.app.state.dag_bag.get_dag(dag_id)
diff --git a/airflow/cli/cli_config.py b/airflow/cli/cli_config.py
index c882804778..04f78f3dc2 100644
--- a/airflow/cli/cli_config.py
+++ b/airflow/cli/cli_config.py
@@ -780,39 +780,39 @@ ARG_INTERNAL_API_ACCESS_LOGFORMAT = Arg(
)
-# ui-api
-ARG_UI_API_PORT = Arg(
+# fastapi-api
+ARG_FASTAPI_API_PORT = Arg(
("-p", "--port"),
default=9091,
type=int,
help="The port on which to run the server",
)
-ARG_UI_API_WORKERS = Arg(
+ARG_FASTAPI_API_WORKERS = Arg(
("-w", "--workers"),
default=4,
type=int,
- help="Number of workers to run the UI API-on",
+ help="Number of workers to run the FastAPI API-on",
)
-ARG_UI_API_WORKER_TIMEOUT = Arg(
+ARG_FASTAPI_API_WORKER_TIMEOUT = Arg(
("-t", "--worker-timeout"),
default=120,
type=int,
- help="The timeout for waiting on UI API workers",
+ help="The timeout for waiting on FastAPI API workers",
)
-ARG_UI_API_HOSTNAME = Arg(
+ARG_FASTAPI_API_HOSTNAME = Arg(
("-H", "--hostname"),
default="0.0.0.0", # nosec
help="Set the hostname on which to run the web server",
)
-ARG_UI_API_ACCESS_LOGFILE = Arg(
+ARG_FASTAPI_API_ACCESS_LOGFILE = Arg(
("-A", "--access-logfile"),
help="The logfile to store the access log. Use '-' to print to stdout",
)
-ARG_UI_API_ERROR_LOGFILE = Arg(
+ARG_FASTAPI_API_ERROR_LOGFILE = Arg(
("-E", "--error-logfile"),
help="The logfile to store the error log. Use '-' to print to stderr",
)
-ARG_UI_API_ACCESS_LOGFORMAT = Arg(
+ARG_FASTAPI_API_ACCESS_LOGFORMAT = Arg(
("-L", "--access-logformat"),
help="The access log format for gunicorn logs",
)
@@ -1981,21 +1981,21 @@ core_commands: list[CLICommand] = [
),
),
ActionCommand(
- name="ui-api",
- help="Start an Airflow UI API instance",
- func=lazy_load_command("airflow.cli.commands.ui_api_command.ui_api"),
+ name="fastapi-api",
+ help="Start an Airflow FastAPI API instance",
+
func=lazy_load_command("airflow.cli.commands.fastapi_api_command.fastapi_api"),
args=(
- ARG_UI_API_PORT,
- ARG_UI_API_WORKERS,
- ARG_UI_API_WORKER_TIMEOUT,
- ARG_UI_API_HOSTNAME,
+ ARG_FASTAPI_API_PORT,
+ ARG_FASTAPI_API_WORKERS,
+ ARG_FASTAPI_API_WORKER_TIMEOUT,
+ ARG_FASTAPI_API_HOSTNAME,
ARG_PID,
ARG_DAEMON,
ARG_STDOUT,
ARG_STDERR,
- ARG_UI_API_ACCESS_LOGFILE,
- ARG_UI_API_ERROR_LOGFILE,
- ARG_UI_API_ACCESS_LOGFORMAT,
+ ARG_FASTAPI_API_ACCESS_LOGFILE,
+ ARG_FASTAPI_API_ERROR_LOGFILE,
+ ARG_FASTAPI_API_ACCESS_LOGFORMAT,
ARG_LOG_FILE,
ARG_SSL_CERT,
ARG_SSL_KEY,
diff --git a/airflow/cli/commands/ui_api_command.py
b/airflow/cli/commands/fastapi_api_command.py
similarity index 91%
rename from airflow/cli/commands/ui_api_command.py
rename to airflow/cli/commands/fastapi_api_command.py
index cacc1fd7d4..d50d454347 100644
--- a/airflow/cli/commands/ui_api_command.py
+++ b/airflow/cli/commands/fastapi_api_command.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""UI API command."""
+"""FastAPI API command."""
from __future__ import annotations
@@ -52,8 +52,8 @@ AirflowUvicornWorker.CONFIG_KWARGS = {"loop": "asyncio",
"http": "auto"}
@cli_utils.action_cli
@providers_configuration_loaded
-def ui_api(args):
- """Start Airflow UI API."""
+def fastapi_api(args):
+ """Start Airflow FastAPI API."""
print(settings.HEADER)
access_logfile = args.access_logfile or "-"
@@ -62,18 +62,18 @@ def ui_api(args):
num_workers = args.workers
worker_timeout = args.worker_timeout
- worker_class = "airflow.cli.commands.ui_api_command.AirflowUvicornWorker"
+ worker_class =
"airflow.cli.commands.fastapi_api_command.AirflowUvicornWorker"
- from airflow.api_ui.app import create_app
+ from airflow.api_fastapi.app import create_app
if args.debug:
- print(f"Starting the UI API server on port {args.port} and host
{args.hostname} debug.")
+ print(f"Starting the FastAPI API server on port {args.port} and host
{args.hostname} debug.")
log.warning("Running in dev mode, ignoring gunicorn args")
run_args = [
"fastapi",
"dev",
- "airflow/api_ui/main.py",
+ "airflow/api_fastapi/main.py",
"--port",
str(args.port),
"--host",
@@ -99,7 +99,7 @@ def ui_api(args):
)
)
- pid_file, _, _, _ = setup_locations("ui-api", pid=args.pid)
+ pid_file, _, _, _ = setup_locations("fastapi-api", pid=args.pid)
run_args = [
sys.executable,
"-m",
@@ -113,7 +113,7 @@ def ui_api(args):
"--bind",
args.hostname + ":" + str(args.port),
"--name",
- "airflow-ui-api",
+ "airflow-fastapi-api",
"--pid",
pid_file,
"--access-logfile",
@@ -121,7 +121,7 @@ def ui_api(args):
"--error-logfile",
str(error_logfile),
"--config",
- "python:airflow.api_ui.gunicorn_config",
+ "python:airflow.api_fastapi.gunicorn_config",
]
if args.access_logformat and args.access_logformat.strip():
@@ -130,7 +130,7 @@ def ui_api(args):
if args.daemon:
run_args += ["--daemon"]
- run_args += ["airflow.api_ui.app:cached_app()"]
+ run_args += ["airflow.api_fastapi.app:cached_app()"]
# To prevent different workers creating the web app and
# all writing to the database at the same time, we use the --preload
option.
@@ -194,7 +194,7 @@ def ui_api(args):
monitor_pid_file =
str(pid_file_path.with_name(f"{pid_file_path.stem}-monitor{pid_file_path.suffix}"))
run_command_with_daemon_option(
args=args,
- process_name="ui-api",
+ process_name="fastapi-api",
callback=lambda: start_and_monitor_gunicorn(args),
should_setup_logging=True,
pid_file=monitor_pid_file,
diff --git a/airflow/cli/commands/standalone_command.py
b/airflow/cli/commands/standalone_command.py
index 1f2bf5c9e9..0f8d45eb5f 100644
--- a/airflow/cli/commands/standalone_command.py
+++ b/airflow/cli/commands/standalone_command.py
@@ -80,10 +80,10 @@ class StandaloneCommand:
command=["webserver"],
env=env,
)
- self.subcommands["ui-api"] = SubCommand(
+ self.subcommands["fastapi-api"] = SubCommand(
self,
- name="ui-api",
- command=["ui-api"],
+ name="fastapi-api",
+ command=["fastapi-api"],
env=env,
)
self.subcommands["triggerer"] = SubCommand(
@@ -142,7 +142,7 @@ class StandaloneCommand:
You can pass multiple lines to output if you wish; it will be split
for you.
"""
color = {
- "ui-api": "magenta",
+ "fastapi-api": "magenta",
"webserver": "green",
"scheduler": "blue",
"triggerer": "cyan",
diff --git a/contributing-docs/03_contributors_quick_start.rst
b/contributing-docs/03_contributors_quick_start.rst
index b7467d6c4a..8f7ead6dea 100644
--- a/contributing-docs/03_contributors_quick_start.rst
+++ b/contributing-docs/03_contributors_quick_start.rst
@@ -335,7 +335,7 @@ Using Breeze
Ports are forwarded to the running docker containers for webserver and
database
* 12322 -> forwarded to Airflow ssh server -> airflow:22
* 28080 -> forwarded to Airflow webserver -> airflow:8080
- * 29091 -> forwarded to Airflow UI API -> airflow:9091
+ * 29091 -> forwarded to Airflow FastAPI API -> airflow:9091
* 25555 -> forwarded to Flower dashboard -> airflow:5555
* 25433 -> forwarded to Postgres database -> postgres:5432
* 23306 -> forwarded to MySQL database -> mysql:3306
@@ -344,7 +344,7 @@ Using Breeze
Here are links to those services that you can use on host:
* ssh connection for remote debugging: ssh -p 12322 airflow@127.0.0.1
(password: airflow)
* Webserver: http://127.0.0.1:28080
- * UI API: http://127.0.0.1:29091
+ * FastAPI API: http://127.0.0.1:29091
* Flower: http://127.0.0.1:25555
* Postgres:
jdbc:postgresql://127.0.0.1:25433/airflow?user=postgres&password=airflow
* Mysql: jdbc:mysql://127.0.0.1:23306/airflow?user=root
diff --git a/contributing-docs/08_static_code_checks.rst
b/contributing-docs/08_static_code_checks.rst
index 0470b0a57d..0f2acf890b 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -272,7 +272,7 @@ require Breeze Docker image to be built locally.
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| generate-airflow-diagrams | Generate airflow
diagrams | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| generate-openapi-spec | Generate the UI
API OPENAPI spec | * |
+| generate-openapi-spec | Generate the
FastAPI API spec | * |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| generate-pypi-readme | Generate PyPI
README | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
diff --git a/contributing-docs/14_node_environment_setup.rst
b/contributing-docs/14_node_environment_setup.rst
index 77ba3e9542..99da685aeb 100644
--- a/contributing-docs/14_node_environment_setup.rst
+++ b/contributing-docs/14_node_environment_setup.rst
@@ -29,9 +29,9 @@ But we want to limit modifications to the legacy
``airflow/www`` views, to mainl
2. The minimum necessary to unblock other Airflow 3.0 feature work
3. Fixes to react views which we haven't migrated over yet, but can still be
ported over to the new UI
-Custom endpoints for the UI will also be moved away from
``airflow/www/views.py`` and to ``airflow/api_ui``.
+Custom endpoints for the UI will also be moved away from
``airflow/www/views.py`` and to ``airflow/api_fastapi``.
Contributions to the legacy views file will follow the same rules.
-Committers will exercise their judgement on what endpoints should exist in the
public ``airflow/api_connexion`` versus the private ``airflow/api_ui``
+Committers will exercise their judgement on what endpoints should exist in the
public ``airflow/api_connexion`` versus the private ``airflow/api_fastapi``
Airflow UI
----------
diff --git a/contributing-docs/testing/unit_tests.rst
b/contributing-docs/testing/unit_tests.rst
index 5c1c2f1584..935a7b9b60 100644
--- a/contributing-docs/testing/unit_tests.rst
+++ b/contributing-docs/testing/unit_tests.rst
@@ -96,7 +96,7 @@ test types you want to use in various ``breeze testing``
sub-commands in three w
Those test types are defined:
* ``Always`` - those are tests that should be always executed (always
sub-folder)
-* ``API`` - Tests for the Airflow API (api, api_connexion, api_internal,
api_ui sub-folders)
+* ``API`` - Tests for the Airflow API (api, api_connexion, api_internal,
api_fastapi sub-folders)
* ``CLI`` - Tests for the Airflow CLI (cli folder)
* ``Core`` - for the core Airflow functionality (core, executors, jobs,
models, ti_deps, utils sub-folders)
* ``Operators`` - tests for the operators (operators folder with exception of
Virtualenv Operator tests and
diff --git a/dev/breeze/doc/03_developer_tasks.rst
b/dev/breeze/doc/03_developer_tasks.rst
index 4acfdb4627..76f4360683 100644
--- a/dev/breeze/doc/03_developer_tasks.rst
+++ b/dev/breeze/doc/03_developer_tasks.rst
@@ -113,7 +113,7 @@ When you run Airflow Breeze, the following ports are
automatically forwarded:
* 12322 -> forwarded to Airflow ssh server -> airflow:22
* 28080 -> forwarded to Airflow webserver -> airflow:8080
- * 29091 -> forwarded to Airflow UI API -> airflow:9091
+ * 29091 -> forwarded to Airflow FastAPI API -> airflow:9091
* 25555 -> forwarded to Flower dashboard -> airflow:5555
* 25433 -> forwarded to Postgres database -> postgres:5432
* 23306 -> forwarded to MySQL database -> mysql:3306
@@ -126,7 +126,7 @@ You can connect to these ports/databases using:
* ssh connection for remote debugging: ssh -p 12322 airflow@127.0.0.1 pw:
airflow
* Webserver: http://127.0.0.1:28080
- * UI API: http://127.0.0.1:29091
+ * FastAPI API: http://127.0.0.1:29091
* Flower: http://127.0.0.1:25555
* Postgres:
jdbc:postgresql://127.0.0.1:25433/airflow?user=postgres&password=airflow
* Mysql: jdbc:mysql://127.0.0.1:23306/airflow?user=root
@@ -156,7 +156,7 @@ You can change the used host port numbers by setting
appropriate environment var
* ``SSH_PORT``
* ``WEBSERVER_HOST_PORT``
-* ``UI_API_HOST_PORT``
+* ``FASTAPI_API_HOST_PORT``
* ``POSTGRES_HOST_PORT``
* ``MYSQL_HOST_PORT``
* ``MSSQL_HOST_PORT``
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 0d6eb5f6f9..4ee452af6d 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -253,7 +253,7 @@ REDIS_HOST_PORT = "26379"
SSH_PORT = "12322"
WEBSERVER_HOST_PORT = "28080"
VITE_DEV_PORT = "5173"
-UI_API_HOST_PORT = "29091"
+FASTAPI_API_HOST_PORT = "29091"
CELERY_BROKER_URLS_MAP = {"rabbitmq": "amqp://guest:guest@rabbitmq:5672",
"redis": "redis://redis:6379/0"}
SQLITE_URL = "sqlite:////root/airflow/sqlite/airflow.db"
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py
b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 2f53c73fc7..af74be27c9 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -39,6 +39,7 @@ from airflow_breeze.global_constants import (
DEFAULT_UV_HTTP_TIMEOUT,
DOCKER_DEFAULT_PLATFORM,
DRILL_HOST_PORT,
+ FASTAPI_API_HOST_PORT,
FLOWER_HOST_PORT,
MOUNT_ALL,
MOUNT_PROVIDERS_AND_TESTS,
@@ -52,7 +53,6 @@ from airflow_breeze.global_constants import (
SSH_PORT,
START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR,
TESTABLE_INTEGRATIONS,
- UI_API_HOST_PORT,
USE_AIRFLOW_MOUNT_SOURCES,
WEBSERVER_HOST_PORT,
GithubEvents,
@@ -575,7 +575,7 @@ class ShellParams:
_set_var(_env, "VERBOSE_COMMANDS", self.verbose_commands)
_set_var(_env, "VERSION_SUFFIX_FOR_PYPI", self.version_suffix_for_pypi)
_set_var(_env, "WEBSERVER_HOST_PORT", None, WEBSERVER_HOST_PORT)
- _set_var(_env, "UI_API_HOST_PORT", None, UI_API_HOST_PORT)
+ _set_var(_env, "FASTAPI_API_HOST_PORT", None, FASTAPI_API_HOST_PORT)
_set_var(_env, "_AIRFLOW_RUN_DB_TESTS_ONLY", self.run_db_tests_only)
_set_var(_env, "_AIRFLOW_SKIP_DB_TESTS", self.skip_db_tests)
self._generate_env_for_docker_compose_file_if_needed(_env)
diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py
b/dev/breeze/src/airflow_breeze/utils/run_tests.py
index de48da86e4..840bf2fcad 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_tests.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py
@@ -133,7 +133,7 @@ def get_excluded_provider_args(python_version: str) ->
list[str]:
TEST_TYPE_MAP_TO_PYTEST_ARGS: dict[str, list[str]] = {
"Always": ["tests/always"],
- "API": ["tests/api", "tests/api_connexion", "tests/api_internal",
"tests/api_ui"],
+ "API": ["tests/api", "tests/api_connexion", "tests/api_internal",
"tests/api_fastapi"],
"BranchPythonVenv": [
"tests/operators/test_python.py::TestBranchPythonVirtualenvOperator",
],
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index eb38675fb3..653d71f9a4 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -264,11 +264,11 @@ TEST_TYPE_MATCHES = HashableDict(
r"^airflow/api/",
r"^airflow/api_connexion/",
r"^airflow/api_internal/",
- r"^airflow/api_ui/",
+ r"^airflow/api_fastapi/",
r"^tests/api/",
r"^tests/api_connexion/",
r"^tests/api_internal/",
- r"^tests/api_ui/",
+ r"^tests/api_fastapi/",
],
SelectiveUnitTestTypes.CLI: [
r"^airflow/cli/",
diff --git a/dev/breeze/src/airflow_breeze/utils/visuals.py
b/dev/breeze/src/airflow_breeze/utils/visuals.py
index a43182713f..b9df215fb1 100644
--- a/dev/breeze/src/airflow_breeze/utils/visuals.py
+++ b/dev/breeze/src/airflow_breeze/utils/visuals.py
@@ -21,12 +21,12 @@ Visuals displayed to the user when entering Breeze shell.
from __future__ import annotations
from airflow_breeze.global_constants import (
+ FASTAPI_API_HOST_PORT,
FLOWER_HOST_PORT,
MYSQL_HOST_PORT,
POSTGRES_HOST_PORT,
REDIS_HOST_PORT,
SSH_PORT,
- UI_API_HOST_PORT,
WEBSERVER_HOST_PORT,
)
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
@@ -83,7 +83,7 @@ CHEATSHEET = f"""
Ports are forwarded to the running docker containers for webserver and
database
* {SSH_PORT} -> forwarded to Airflow ssh server -> airflow:22
* {WEBSERVER_HOST_PORT} -> forwarded to Airflow webserver ->
airflow:8080
- * {UI_API_HOST_PORT} -> forwarded to Airflow UI API -> airflow:9091
+ * {FASTAPI_API_HOST_PORT} -> forwarded to Airflow FastAPI API ->
airflow:9091
* {FLOWER_HOST_PORT} -> forwarded to Flower dashboard -> airflow:5555
* {POSTGRES_HOST_PORT} -> forwarded to Postgres database ->
postgres:5432
* {MYSQL_HOST_PORT} -> forwarded to MySQL database -> mysql:3306
@@ -93,7 +93,7 @@ CHEATSHEET = f"""
* ssh connection for remote debugging: ssh -p {SSH_PORT}
airflow@127.0.0.1 (password: airflow)
* Webserver: http://127.0.0.1:{WEBSERVER_HOST_PORT}
- * UI API: http://127.0.0.1:{WEBSERVER_HOST_PORT}
+ * FastAPI API: http://127.0.0.1:{WEBSERVER_HOST_PORT}
* Flower: http://127.0.0.1:{FLOWER_HOST_PORT}
* Postgres:
jdbc:postgresql://127.0.0.1:{POSTGRES_HOST_PORT}/airflow?user=postgres&password=airflow
* Mysql:
jdbc:mysql://127.0.0.1:{MYSQL_HOST_PORT}/airflow?user=root
diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py
b/dev/breeze/tests/test_pytest_args_for_test_types.py
index 489dfed86d..36a4b15779 100644
--- a/dev/breeze/tests/test_pytest_args_for_test_types.py
+++ b/dev/breeze/tests/test_pytest_args_for_test_types.py
@@ -54,7 +54,7 @@ from airflow_breeze.utils.run_tests import
convert_parallel_types_to_folders, co
),
(
"API",
- ["tests/api", "tests/api_connexion", "tests/api_internal",
"tests/api_ui"],
+ ["tests/api", "tests/api_connexion", "tests/api_internal",
"tests/api_fastapi"],
False,
),
(
@@ -234,7 +234,7 @@ def test_pytest_args_for_helm_test_types(helm_test_package:
str, pytest_args: li
[
(
"API",
- ["tests/api", "tests/api_connexion", "tests/api_internal",
"tests/api_ui"],
+ ["tests/api", "tests/api_connexion", "tests/api_internal",
"tests/api_fastapi"],
False,
),
(
@@ -250,7 +250,7 @@ def test_pytest_args_for_helm_test_types(helm_test_package:
str, pytest_args: li
"tests/api",
"tests/api_connexion",
"tests/api_internal",
- "tests/api_ui",
+ "tests/api_fastapi",
"tests/cli",
],
False,
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 2491b6f920..ac03f57ba3 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -184,7 +184,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
),
(
pytest.param(
- ("airflow/api_ui/file.py",),
+ ("airflow/api_fastapi/file.py",),
{
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 2556113d00..c7a7d1221c 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -600,6 +600,7 @@ falsy
faq
Fargate
fargate
+fastapi
fbee
fc
fd
diff --git a/scripts/ci/docker-compose/base-ports.yml
b/scripts/ci/docker-compose/base-ports.yml
index 050aaad2dc..17a935cd27 100644
--- a/scripts/ci/docker-compose/base-ports.yml
+++ b/scripts/ci/docker-compose/base-ports.yml
@@ -20,5 +20,5 @@ services:
ports:
- "${SSH_PORT}:22"
- "${WEBSERVER_HOST_PORT}:8080"
- - "${UI_API_HOST_PORT}:9091"
+ - "${FASTAPI_API_HOST_PORT}:9091"
- "${FLOWER_HOST_PORT}:5555"
diff --git a/scripts/ci/pre_commit/check_tests_in_right_folders.py
b/scripts/ci/pre_commit/check_tests_in_right_folders.py
index 5a7e2c4ec2..8260b6ad0d 100755
--- a/scripts/ci/pre_commit/check_tests_in_right_folders.py
+++ b/scripts/ci/pre_commit/check_tests_in_right_folders.py
@@ -33,7 +33,7 @@ POSSIBLE_TEST_FOLDERS = [
"api",
"api_connexion",
"api_internal",
- "api_ui",
+ "api_fastapi",
"auth",
"callbacks",
"charts",
diff --git a/scripts/ci/pre_commit/update_ui_api_spec.py
b/scripts/ci/pre_commit/update_fastapi_api_spec.py
similarity index 94%
rename from scripts/ci/pre_commit/update_ui_api_spec.py
rename to scripts/ci/pre_commit/update_fastapi_api_spec.py
index 9bba385301..15ccaa5ac2 100755
--- a/scripts/ci/pre_commit/update_ui_api_spec.py
+++ b/scripts/ci/pre_commit/update_fastapi_api_spec.py
@@ -26,7 +26,7 @@ from common_precommit_utils import console,
initialize_breeze_precommit, run_com
initialize_breeze_precommit(__name__, __file__)
cmd_result = run_command_via_breeze_shell(
- ["python3", "/opt/airflow/scripts/in_container/run_update_ui_api_spec.py"],
+ ["python3",
"/opt/airflow/scripts/in_container/run_update_fastapi_api_spec.py"],
backend="postgres",
skip_environment_initialization=False,
)
diff --git a/scripts/in_container/bin/run_tmux
b/scripts/in_container/bin/run_tmux
index fce52fc2d2..61e7dbf49d 100755
--- a/scripts/in_container/bin/run_tmux
+++ b/scripts/in_container/bin/run_tmux
@@ -60,9 +60,9 @@ tmux send-keys 'airflow scheduler' C-m
tmux select-pane -t 2
tmux split-window -h
if [[ ${DEV_MODE=} == "true" ]]; then
- tmux send-keys 'airflow ui-api -d' C-m
+ tmux send-keys 'airflow fastapi-api -d' C-m
else
- tmux send-keys 'airflow ui-api' C-m
+ tmux send-keys 'airflow fastapi-api' C-m
fi
tmux split-window -h
diff --git a/scripts/in_container/run_update_ui_api_spec.py
b/scripts/in_container/run_update_fastapi_api_spec.py
similarity index 79%
rename from scripts/in_container/run_update_ui_api_spec.py
rename to scripts/in_container/run_update_fastapi_api_spec.py
index c21b7905bf..4d78bc4afd 100644
--- a/scripts/in_container/run_update_ui_api_spec.py
+++ b/scripts/in_container/run_update_fastapi_api_spec.py
@@ -19,11 +19,17 @@ from __future__ import annotations
import yaml
from fastapi.openapi.utils import get_openapi
-from airflow.api_ui.app import cached_app
+from airflow.api_fastapi.app import create_app
-app = cached_app()
+app = create_app()
-OPENAPI_SPEC_FILE = "airflow/api_ui/openapi/v1-generated.yaml"
+OPENAPI_SPEC_FILE = "airflow/api_fastapi/openapi/v1-generated.yaml"
+
+
+# The persisted openapi spec will list all endpoints (public and ui), this
+# is used for code generation.
+for route in app.routes:
+ route.__setattr__("include_in_schema", True)
with open(OPENAPI_SPEC_FILE, "w+") as f:
yaml.dump(
diff --git a/tests/api_ui/__init__.py b/tests/api_fastapi/__init__.py
similarity index 100%
rename from tests/api_ui/__init__.py
rename to tests/api_fastapi/__init__.py
diff --git a/tests/api_ui/conftest.py b/tests/api_fastapi/conftest.py
similarity index 95%
rename from tests/api_ui/conftest.py
rename to tests/api_fastapi/conftest.py
index 9f82802142..c5212272d7 100644
--- a/tests/api_ui/conftest.py
+++ b/tests/api_fastapi/conftest.py
@@ -19,7 +19,7 @@ from __future__ import annotations
import pytest
from fastapi.testclient import TestClient
-from airflow.api_ui.app import create_app
+from airflow.api_fastapi.app import create_app
@pytest.fixture
diff --git a/tests/api_ui/views/__init__.py
b/tests/api_fastapi/views/__init__.py
similarity index 100%
rename from tests/api_ui/views/__init__.py
rename to tests/api_fastapi/views/__init__.py
diff --git a/airflow/api_ui/__init__.py b/tests/api_fastapi/views/ui/__init__.py
similarity index 100%
rename from airflow/api_ui/__init__.py
rename to tests/api_fastapi/views/ui/__init__.py
diff --git a/tests/api_ui/views/test_datasets.py
b/tests/api_fastapi/views/ui/test_datasets.py
similarity index 100%
rename from tests/api_ui/views/test_datasets.py
rename to tests/api_fastapi/views/ui/test_datasets.py
diff --git a/tests/cli/commands/test_ui_api_command.py
b/tests/cli/commands/test_fastapi_api_command.py
similarity index 68%
rename from tests/cli/commands/test_ui_api_command.py
rename to tests/cli/commands/test_fastapi_api_command.py
index 81ee2275bf..529c67f5ed 100644
--- a/tests/cli/commands/test_ui_api_command.py
+++ b/tests/cli/commands/test_fastapi_api_command.py
@@ -26,7 +26,7 @@ import psutil
import pytest
from rich.console import Console
-from airflow.cli.commands import ui_api_command
+from airflow.cli.commands import fastapi_api_command
from tests.cli.commands._common_cli_classes import _CommonCLIGunicornTestClass
console = Console(width=400, color_system="standard")
@@ -34,28 +34,28 @@ console = Console(width=400, color_system="standard")
@pytest.mark.db_test
class TestCliInternalAPI(_CommonCLIGunicornTestClass):
- main_process_regexp = r"airflow ui-api"
+ main_process_regexp = r"airflow fastapi-api"
@pytest.mark.execution_timeout(210)
- def test_cli_ui_api_background(self, tmp_path):
+ def test_cli_fastapi_api_background(self, tmp_path):
parent_path = tmp_path / "gunicorn"
parent_path.mkdir()
- pidfile_ui_api = parent_path / "pidflow-ui-api.pid"
- pidfile_monitor = parent_path / "pidflow-ui-api-monitor.pid"
- stdout = parent_path / "airflow-ui-api.out"
- stderr = parent_path / "airflow-ui-api.err"
- logfile = parent_path / "airflow-ui-api.log"
+ pidfile_fastapi_api = parent_path / "pidflow-fastapi-api.pid"
+ pidfile_monitor = parent_path / "pidflow-fastapi-api-monitor.pid"
+ stdout = parent_path / "airflow-fastapi-api.out"
+ stderr = parent_path / "airflow-fastapi-api.err"
+ logfile = parent_path / "airflow-fastapi-api.log"
try:
# Run internal-api as daemon in background. Note that the wait
method is not called.
- console.print("[magenta]Starting airflow ui-api --daemon")
+ console.print("[magenta]Starting airflow fastapi-api --daemon")
env = os.environ.copy()
proc = subprocess.Popen(
[
"airflow",
- "ui-api",
+ "fastapi-api",
"--daemon",
"--pid",
- os.fspath(pidfile_ui_api),
+ os.fspath(pidfile_fastapi_api),
"--stdout",
os.fspath(stdout),
"--stderr",
@@ -69,11 +69,11 @@ class TestCliInternalAPI(_CommonCLIGunicornTestClass):
pid_monitor = self._wait_pidfile(pidfile_monitor)
console.print(f"[blue]Monitor started at {pid_monitor}")
- pid_ui_api = self._wait_pidfile(pidfile_ui_api)
- console.print(f"[blue]UI API started at {pid_ui_api}")
- console.print("[blue]Running airflow ui-api process:")
- # Assert that the ui-api and gunicorn processes are running (by
name rather than pid).
- assert self._find_process(r"airflow ui-api --daemon",
print_found_process=True)
+ pid_fastapi_api = self._wait_pidfile(pidfile_fastapi_api)
+ console.print(f"[blue]FastAPI API started at {pid_fastapi_api}")
+ console.print("[blue]Running airflow fastapi-api process:")
+ # Assert that the fastapi-api and gunicorn processes are running
(by name rather than pid).
+ assert self._find_process(r"airflow fastapi-api --daemon",
print_found_process=True)
console.print("[blue]Waiting for gunicorn processes:")
# wait for gunicorn to start
for _ in range(30):
@@ -83,16 +83,16 @@ class TestCliInternalAPI(_CommonCLIGunicornTestClass):
time.sleep(1)
console.print("[blue]Running gunicorn processes:")
assert self._find_all_processes("^gunicorn",
print_found_process=True)
- console.print("[magenta]ui-api process started successfully.")
+ console.print("[magenta]fastapi-api process started successfully.")
console.print(
"[magenta]Terminating monitor process and expect "
- "ui-api and gunicorn processes to terminate as well"
+ "fastapi-api and gunicorn processes to terminate as well"
)
proc = psutil.Process(pid_monitor)
proc.terminate()
assert proc.wait(120) in (0, None)
self._check_processes(ignore_running=False)
- console.print("[magenta]All ui-api and gunicorn processes are
terminated.")
+ console.print("[magenta]All fastapi-api and gunicorn processes are
terminated.")
except Exception:
console.print("[red]Exception occurred. Dumping all logs.")
# Dump all logs
@@ -101,18 +101,20 @@ class TestCliInternalAPI(_CommonCLIGunicornTestClass):
console.print(file.read_text())
raise
- def test_cli_ui_api_debug(self, app):
- with mock.patch("subprocess.Popen") as Popen,
mock.patch.object(ui_api_command, "GunicornMonitor"):
+ def test_cli_fastapi_api_debug(self, app):
+ with mock.patch("subprocess.Popen") as Popen, mock.patch.object(
+ fastapi_api_command, "GunicornMonitor"
+ ):
port = "9092"
hostname = "somehost"
- args = self.parser.parse_args(["ui-api", "--port", port,
"--hostname", hostname, "--debug"])
- ui_api_command.ui_api(args)
+ args = self.parser.parse_args(["fastapi-api", "--port", port,
"--hostname", hostname, "--debug"])
+ fastapi_api_command.fastapi_api(args)
Popen.assert_called_with(
[
"fastapi",
"dev",
- "airflow/api_ui/main.py",
+ "airflow/api_fastapi/main.py",
"--port",
port,
"--host",
@@ -121,18 +123,20 @@ class TestCliInternalAPI(_CommonCLIGunicornTestClass):
close_fds=True,
)
- def test_cli_ui_api_args(self):
- with mock.patch("subprocess.Popen") as Popen,
mock.patch.object(ui_api_command, "GunicornMonitor"):
+ def test_cli_fastapi_api_args(self):
+ with mock.patch("subprocess.Popen") as Popen, mock.patch.object(
+ fastapi_api_command, "GunicornMonitor"
+ ):
args = self.parser.parse_args(
[
- "ui-api",
+ "fastapi-api",
"--access-logformat",
"custom_log_format",
"--pid",
"/tmp/x.pid",
]
)
- ui_api_command.ui_api(args)
+ fastapi_api_command.fastapi_api(args)
Popen.assert_called_with(
[
@@ -142,13 +146,13 @@ class TestCliInternalAPI(_CommonCLIGunicornTestClass):
"--workers",
"4",
"--worker-class",
- "airflow.cli.commands.ui_api_command.AirflowUvicornWorker",
+
"airflow.cli.commands.fastapi_api_command.AirflowUvicornWorker",
"--timeout",
"120",
"--bind",
"0.0.0.0:9091",
"--name",
- "airflow-ui-api",
+ "airflow-fastapi-api",
"--pid",
"/tmp/x.pid",
"--access-logfile",
@@ -156,10 +160,10 @@ class TestCliInternalAPI(_CommonCLIGunicornTestClass):
"--error-logfile",
"-",
"--config",
- "python:airflow.api_ui.gunicorn_config",
+ "python:airflow.api_fastapi.gunicorn_config",
"--access-logformat",
"custom_log_format",
- "airflow.api_ui.app:cached_app()",
+ "airflow.api_fastapi.app:cached_app()",
"--preload",
],
close_fds=True,
