This is an automated email from the ASF dual-hosted git repository.
pierrejeambrun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6ad428875e AIP-84 Get Variables (#43083)
6ad428875e is described below
commit 6ad428875e57e2d90678e657608d2d43c86b00c0
Author: Pierre Jeambrun <[email protected]>
AuthorDate: Fri Oct 18 22:23:47 2024 +0800
AIP-84 Get Variables (#43083)
* AIP-84 Get Variables
* Remove value sorting for Variable
---
.../api_connexion/endpoints/variable_endpoint.py | 1 +
.../api_fastapi/core_api/openapi/v1-generated.yaml | 75 +++++++++++++++++++++-
.../core_api/routes/public/variables.py | 45 ++++++++++++-
.../api_fastapi/core_api/serializers/variables.py | 7 ++
airflow/ui/openapi-gen/queries/common.ts | 23 +++++++
airflow/ui/openapi-gen/queries/prefetch.ts | 30 +++++++++
airflow/ui/openapi-gen/queries/queries.ts | 36 +++++++++++
airflow/ui/openapi-gen/queries/suspense.ts | 36 +++++++++++
airflow/ui/openapi-gen/requests/schemas.gen.ts | 20 ++++++
airflow/ui/openapi-gen/requests/services.gen.ts | 31 +++++++++
airflow/ui/openapi-gen/requests/types.gen.ts | 37 +++++++++++
.../core_api/routes/public/test_variables.py | 42 ++++++++++--
12 files changed, 371 insertions(+), 12 deletions(-)
diff --git a/airflow/api_connexion/endpoints/variable_endpoint.py
b/airflow/api_connexion/endpoints/variable_endpoint.py
index 20e7ce1ede..00f5abf00b 100644
--- a/airflow/api_connexion/endpoints/variable_endpoint.py
+++ b/airflow/api_connexion/endpoints/variable_endpoint.py
@@ -70,6 +70,7 @@ def get_variable(*, variable_key: str, session: Session =
NEW_SESSION) -> Respon
return variable_schema.dump(var)
+@mark_fastapi_migration_done
@security.requires_access_variable("GET")
@format_parameters({"limit": check_limit})
@provide_session
diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
index d7ee6340c6..88e2923647 100644
--- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
+++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml
@@ -805,6 +805,59 @@ paths:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/public/variables/:
+ get:
+ tags:
+ - Variable
+ summary: Get Variables
+ description: Get all Variables entries.
+ operationId: get_variables
+ parameters:
+ - name: limit
+ in: query
+ required: false
+ schema:
+ type: integer
+ default: 100
+ title: Limit
+ - name: offset
+ in: query
+ required: false
+ schema:
+ type: integer
+ default: 0
+ title: Offset
+ - name: order_by
+ in: query
+ required: false
+ schema:
+ type: string
+ default: id
+ title: Order By
+ responses:
+ '200':
+ description: Successful Response
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/VariableCollectionResponse'
+ '401':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPExceptionResponse'
+ description: Unauthorized
+ '403':
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPExceptionResponse'
+ description: Forbidden
+ '422':
+ description: Validation Error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/HTTPValidationError'
post:
tags:
- Variable
@@ -812,11 +865,11 @@ paths:
description: Create a variable.
operationId: post_variable
requestBody:
+ required: true
content:
application/json:
schema:
$ref: '#/components/schemas/VariableBody'
- required: true
responses:
'201':
description: Successful Response
@@ -825,17 +878,17 @@ paths:
schema:
$ref: '#/components/schemas/VariableResponse'
'401':
- description: Unauthorized
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPExceptionResponse'
+ description: Unauthorized
'403':
- description: Forbidden
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPExceptionResponse'
+ description: Forbidden
'422':
description: Validation Error
content:
@@ -1840,6 +1893,22 @@ components:
- value
title: VariableBody
description: Variable serializer for bodies.
+ VariableCollectionResponse:
+ properties:
+ variables:
+ items:
+ $ref: '#/components/schemas/VariableResponse'
+ type: array
+ title: Variables
+ total_entries:
+ type: integer
+ title: Total Entries
+ type: object
+ required:
+ - variables
+ - total_entries
+ title: VariableCollectionResponse
+ description: Variable Collection serializer for responses.
VariableResponse:
properties:
key:
diff --git a/airflow/api_fastapi/core_api/routes/public/variables.py
b/airflow/api_fastapi/core_api/routes/public/variables.py
index 3a46c519c9..3110512c2c 100644
--- a/airflow/api_fastapi/core_api/routes/public/variables.py
+++ b/airflow/api_fastapi/core_api/routes/public/variables.py
@@ -21,10 +21,15 @@ from sqlalchemy import select
from sqlalchemy.orm import Session
from typing_extensions import Annotated
-from airflow.api_fastapi.common.db.common import get_session
+from airflow.api_fastapi.common.db.common import get_session, paginated_select
+from airflow.api_fastapi.common.parameters import QueryLimit, QueryOffset,
SortParam
from airflow.api_fastapi.common.router import AirflowRouter
from airflow.api_fastapi.core_api.openapi.exceptions import
create_openapi_http_exception_doc
-from airflow.api_fastapi.core_api.serializers.variables import VariableBody,
VariableResponse
+from airflow.api_fastapi.core_api.serializers.variables import (
+ VariableBody,
+ VariableCollectionResponse,
+ VariableResponse,
+)
from airflow.models.variable import Variable
variables_router = AirflowRouter(tags=["Variable"], prefix="/variables")
@@ -58,6 +63,42 @@ async def get_variable(
return VariableResponse.model_validate(variable, from_attributes=True)
+@variables_router.get(
+ "/",
+ responses=create_openapi_http_exception_doc([401, 403]),
+)
+async def get_variables(
+ limit: QueryLimit,
+ offset: QueryOffset,
+ order_by: Annotated[
+ SortParam,
+ Depends(
+ SortParam(
+ ["key", "id"],
+ Variable,
+ ).dynamic_depends()
+ ),
+ ],
+ session: Annotated[Session, Depends(get_session)],
+) -> VariableCollectionResponse:
+ """Get all Variables entries."""
+ variable_select, total_entries = paginated_select(
+ select(Variable),
+ [],
+ order_by=order_by,
+ offset=offset,
+ limit=limit,
+ session=session,
+ )
+
+ variables = session.scalars(variable_select).all()
+
+ return VariableCollectionResponse(
+ variables=[VariableResponse.model_validate(variable,
from_attributes=True) for variable in variables],
+ total_entries=total_entries,
+ )
+
+
@variables_router.patch("/{variable_key}",
responses=create_openapi_http_exception_doc([400, 401, 403, 404]))
async def patch_variable(
variable_key: str,
diff --git a/airflow/api_fastapi/core_api/serializers/variables.py
b/airflow/api_fastapi/core_api/serializers/variables.py
index 1ecc87425a..b328972544 100644
--- a/airflow/api_fastapi/core_api/serializers/variables.py
+++ b/airflow/api_fastapi/core_api/serializers/variables.py
@@ -58,3 +58,10 @@ class VariableBody(VariableBase):
"""Variable serializer for bodies."""
value: str | None
+
+
+class VariableCollectionResponse(BaseModel):
+ """Variable Collection serializer for responses."""
+
+ variables: list[VariableResponse]
+ total_entries: int
diff --git a/airflow/ui/openapi-gen/queries/common.ts
b/airflow/ui/openapi-gen/queries/common.ts
index 2a05104c8f..7c0ab1c646 100644
--- a/airflow/ui/openapi-gen/queries/common.ts
+++ b/airflow/ui/openapi-gen/queries/common.ts
@@ -192,6 +192,29 @@ export const UseVariableServiceGetVariableKeyFn = (
},
queryKey?: Array<unknown>,
) => [useVariableServiceGetVariableKey, ...(queryKey ?? [{ variableKey }])];
+export type VariableServiceGetVariablesDefaultResponse = Awaited<
+ ReturnType<typeof VariableService.getVariables>
+>;
+export type VariableServiceGetVariablesQueryResult<
+ TData = VariableServiceGetVariablesDefaultResponse,
+ TError = unknown,
+> = UseQueryResult<TData, TError>;
+export const useVariableServiceGetVariablesKey = "VariableServiceGetVariables";
+export const UseVariableServiceGetVariablesKeyFn = (
+ {
+ limit,
+ offset,
+ orderBy,
+ }: {
+ limit?: number;
+ offset?: number;
+ orderBy?: string;
+ } = {},
+ queryKey?: Array<unknown>,
+) => [
+ useVariableServiceGetVariablesKey,
+ ...(queryKey ?? [{ limit, offset, orderBy }]),
+];
export type DagRunServiceGetDagRunDefaultResponse = Awaited<
ReturnType<typeof DagRunService.getDagRun>
>;
diff --git a/airflow/ui/openapi-gen/queries/prefetch.ts
b/airflow/ui/openapi-gen/queries/prefetch.ts
index 28145dc536..98c706a63f 100644
--- a/airflow/ui/openapi-gen/queries/prefetch.ts
+++ b/airflow/ui/openapi-gen/queries/prefetch.ts
@@ -238,6 +238,36 @@ export const prefetchUseVariableServiceGetVariable = (
queryKey: Common.UseVariableServiceGetVariableKeyFn({ variableKey }),
queryFn: () => VariableService.getVariable({ variableKey }),
});
+/**
+ * Get Variables
+ * Get all Variables entries.
+ * @param data The data for the request.
+ * @param data.limit
+ * @param data.offset
+ * @param data.orderBy
+ * @returns VariableCollectionResponse Successful Response
+ * @throws ApiError
+ */
+export const prefetchUseVariableServiceGetVariables = (
+ queryClient: QueryClient,
+ {
+ limit,
+ offset,
+ orderBy,
+ }: {
+ limit?: number;
+ offset?: number;
+ orderBy?: string;
+ } = {},
+) =>
+ queryClient.prefetchQuery({
+ queryKey: Common.UseVariableServiceGetVariablesKeyFn({
+ limit,
+ offset,
+ orderBy,
+ }),
+ queryFn: () => VariableService.getVariables({ limit, offset, orderBy }),
+ });
/**
* Get Dag Run
* @param data The data for the request.
diff --git a/airflow/ui/openapi-gen/queries/queries.ts
b/airflow/ui/openapi-gen/queries/queries.ts
index 7009a7dee7..9d3e640fbf 100644
--- a/airflow/ui/openapi-gen/queries/queries.ts
+++ b/airflow/ui/openapi-gen/queries/queries.ts
@@ -301,6 +301,42 @@ export const useVariableServiceGetVariable = <
queryFn: () => VariableService.getVariable({ variableKey }) as TData,
...options,
});
+/**
+ * Get Variables
+ * Get all Variables entries.
+ * @param data The data for the request.
+ * @param data.limit
+ * @param data.offset
+ * @param data.orderBy
+ * @returns VariableCollectionResponse Successful Response
+ * @throws ApiError
+ */
+export const useVariableServiceGetVariables = <
+ TData = Common.VariableServiceGetVariablesDefaultResponse,
+ TError = unknown,
+ TQueryKey extends Array<unknown> = unknown[],
+>(
+ {
+ limit,
+ offset,
+ orderBy,
+ }: {
+ limit?: number;
+ offset?: number;
+ orderBy?: string;
+ } = {},
+ queryKey?: TQueryKey,
+ options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">,
+) =>
+ useQuery<TData, TError>({
+ queryKey: Common.UseVariableServiceGetVariablesKeyFn(
+ { limit, offset, orderBy },
+ queryKey,
+ ),
+ queryFn: () =>
+ VariableService.getVariables({ limit, offset, orderBy }) as TData,
+ ...options,
+ });
/**
* Get Dag Run
* @param data The data for the request.
diff --git a/airflow/ui/openapi-gen/queries/suspense.ts
b/airflow/ui/openapi-gen/queries/suspense.ts
index 51e35f321a..d3d8b7b344 100644
--- a/airflow/ui/openapi-gen/queries/suspense.ts
+++ b/airflow/ui/openapi-gen/queries/suspense.ts
@@ -296,6 +296,42 @@ export const useVariableServiceGetVariableSuspense = <
queryFn: () => VariableService.getVariable({ variableKey }) as TData,
...options,
});
+/**
+ * Get Variables
+ * Get all Variables entries.
+ * @param data The data for the request.
+ * @param data.limit
+ * @param data.offset
+ * @param data.orderBy
+ * @returns VariableCollectionResponse Successful Response
+ * @throws ApiError
+ */
+export const useVariableServiceGetVariablesSuspense = <
+ TData = Common.VariableServiceGetVariablesDefaultResponse,
+ TError = unknown,
+ TQueryKey extends Array<unknown> = unknown[],
+>(
+ {
+ limit,
+ offset,
+ orderBy,
+ }: {
+ limit?: number;
+ offset?: number;
+ orderBy?: string;
+ } = {},
+ queryKey?: TQueryKey,
+ options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">,
+) =>
+ useSuspenseQuery<TData, TError>({
+ queryKey: Common.UseVariableServiceGetVariablesKeyFn(
+ { limit, offset, orderBy },
+ queryKey,
+ ),
+ queryFn: () =>
+ VariableService.getVariables({ limit, offset, orderBy }) as TData,
+ ...options,
+ });
/**
* Get Dag Run
* @param data The data for the request.
diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts
b/airflow/ui/openapi-gen/requests/schemas.gen.ts
index 981c8f659a..f2214e2e30 100644
--- a/airflow/ui/openapi-gen/requests/schemas.gen.ts
+++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts
@@ -1371,6 +1371,26 @@ export const $VariableBody = {
description: "Variable serializer for bodies.",
} as const;
+export const $VariableCollectionResponse = {
+ properties: {
+ variables: {
+ items: {
+ $ref: "#/components/schemas/VariableResponse",
+ },
+ type: "array",
+ title: "Variables",
+ },
+ total_entries: {
+ type: "integer",
+ title: "Total Entries",
+ },
+ },
+ type: "object",
+ required: ["variables", "total_entries"],
+ title: "VariableCollectionResponse",
+ description: "Variable Collection serializer for responses.",
+} as const;
+
export const $VariableResponse = {
properties: {
key: {
diff --git a/airflow/ui/openapi-gen/requests/services.gen.ts
b/airflow/ui/openapi-gen/requests/services.gen.ts
index 7516a625d7..1561c024a8 100644
--- a/airflow/ui/openapi-gen/requests/services.gen.ts
+++ b/airflow/ui/openapi-gen/requests/services.gen.ts
@@ -31,6 +31,8 @@ import type {
GetVariableResponse,
PatchVariableData,
PatchVariableResponse,
+ GetVariablesData,
+ GetVariablesResponse,
PostVariableData,
PostVariableResponse,
GetDagRunData,
@@ -465,6 +467,35 @@ export class VariableService {
});
}
+ /**
+ * Get Variables
+ * Get all Variables entries.
+ * @param data The data for the request.
+ * @param data.limit
+ * @param data.offset
+ * @param data.orderBy
+ * @returns VariableCollectionResponse Successful Response
+ * @throws ApiError
+ */
+ public static getVariables(
+ data: GetVariablesData = {},
+ ): CancelablePromise<GetVariablesResponse> {
+ return __request(OpenAPI, {
+ method: "GET",
+ url: "/public/variables/",
+ query: {
+ limit: data.limit,
+ offset: data.offset,
+ order_by: data.orderBy,
+ },
+ errors: {
+ 401: "Unauthorized",
+ 403: "Forbidden",
+ 422: "Validation Error",
+ },
+ });
+ }
+
/**
* Post Variable
* Create a variable.
diff --git a/airflow/ui/openapi-gen/requests/types.gen.ts
b/airflow/ui/openapi-gen/requests/types.gen.ts
index 5f5ce6da71..aae457b13a 100644
--- a/airflow/ui/openapi-gen/requests/types.gen.ts
+++ b/airflow/ui/openapi-gen/requests/types.gen.ts
@@ -308,6 +308,14 @@ export type VariableBody = {
value: string | null;
};
+/**
+ * Variable Collection serializer for responses.
+ */
+export type VariableCollectionResponse = {
+ variables: Array<VariableResponse>;
+ total_entries: number;
+};
+
/**
* Variable serializer for responses.
*/
@@ -428,6 +436,14 @@ export type PatchVariableData = {
export type PatchVariableResponse = VariableResponse;
+export type GetVariablesData = {
+ limit?: number;
+ offset?: number;
+ orderBy?: string;
+};
+
+export type GetVariablesResponse = VariableCollectionResponse;
+
export type PostVariableData = {
requestBody: VariableBody;
};
@@ -812,6 +828,27 @@ export type $OpenApiTs = {
};
};
"/public/variables/": {
+ get: {
+ req: GetVariablesData;
+ res: {
+ /**
+ * Successful Response
+ */
+ 200: VariableCollectionResponse;
+ /**
+ * Unauthorized
+ */
+ 401: HTTPExceptionResponse;
+ /**
+ * Forbidden
+ */
+ 403: HTTPExceptionResponse;
+ /**
+ * Validation Error
+ */
+ 422: HTTPValidationError;
+ };
+ };
post: {
req: PostVariableData;
res: {
diff --git a/tests/api_fastapi/core_api/routes/public/test_variables.py
b/tests/api_fastapi/core_api/routes/public/test_variables.py
index 0a52d19710..8e412c94cb 100644
--- a/tests/api_fastapi/core_api/routes/public/test_variables.py
+++ b/tests/api_fastapi/core_api/routes/public/test_variables.py
@@ -41,7 +41,7 @@ TEST_VARIABLE_DESCRIPTION3 = "Some description for the
variable"
@provide_session
-def _create_variable(session) -> None:
+def _create_variables(session) -> None:
Variable.set(
key=TEST_VARIABLE_KEY,
value=TEST_VARIABLE_VALUE,
@@ -72,13 +72,13 @@ class TestVariableEndpoint:
def teardown_method(self) -> None:
clear_db_variables()
- def create_variable(self):
- _create_variable()
+ def create_variables(self):
+ _create_variables()
class TestDeleteVariable(TestVariableEndpoint):
def test_delete_should_respond_204(self, test_client, session):
- self.create_variable()
+ self.create_variables()
variables = session.query(Variable).all()
assert len(variables) == 3
response = test_client.delete(f"/public/variables/{TEST_VARIABLE_KEY}")
@@ -125,7 +125,7 @@ class TestGetVariable(TestVariableEndpoint):
],
)
def test_get_should_respond_200(self, test_client, session, key,
expected_response):
- self.create_variable()
+ self.create_variables()
response = test_client.get(f"/public/variables/{key}")
assert response.status_code == 200
assert response.json() == expected_response
@@ -137,6 +137,34 @@ class TestGetVariable(TestVariableEndpoint):
assert f"The Variable with key: `{TEST_VARIABLE_KEY}` was not found"
== body["detail"]
+class TestGetVariables(TestVariableEndpoint):
+ @pytest.mark.enable_redact
+ @pytest.mark.parametrize(
+ "query_params, expected_total_entries, expected_keys",
+ [
+ # Filters
+ ({}, 3, [TEST_VARIABLE_KEY, TEST_VARIABLE_KEY2,
TEST_VARIABLE_KEY3]),
+ ({"limit": 1}, 3, [TEST_VARIABLE_KEY]),
+ ({"limit": 1, "offset": 1}, 3, [TEST_VARIABLE_KEY2]),
+ # Sort
+ ({"order_by": "id"}, 3, [TEST_VARIABLE_KEY, TEST_VARIABLE_KEY2,
TEST_VARIABLE_KEY3]),
+ ({"order_by": "-id"}, 3, [TEST_VARIABLE_KEY3, TEST_VARIABLE_KEY2,
TEST_VARIABLE_KEY]),
+ ({"order_by": "key"}, 3, [TEST_VARIABLE_KEY3, TEST_VARIABLE_KEY2,
TEST_VARIABLE_KEY]),
+ ({"order_by": "-key"}, 3, [TEST_VARIABLE_KEY, TEST_VARIABLE_KEY2,
TEST_VARIABLE_KEY3]),
+ ],
+ )
+ def test_should_respond_200(
+ self, session, test_client, query_params, expected_total_entries,
expected_keys
+ ):
+ self.create_variables()
+ response = test_client.get("/public/variables/", params=query_params)
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body["total_entries"] == expected_total_entries
+ assert [variable["key"] for variable in body["variables"]] ==
expected_keys
+
+
class TestPatchVariable(TestVariableEndpoint):
@pytest.mark.enable_redact
@pytest.mark.parametrize(
@@ -201,7 +229,7 @@ class TestPatchVariable(TestVariableEndpoint):
],
)
def test_patch_should_respond_200(self, test_client, session, key, body,
params, expected_response):
- self.create_variable()
+ self.create_variables()
response = test_client.patch(f"/public/variables/{key}", json=body,
params=params)
assert response.status_code == 200
assert response.json() == expected_response
@@ -269,7 +297,7 @@ class TestPostVariable(TestVariableEndpoint):
],
)
def test_post_should_respond_201(self, test_client, session, body,
expected_response):
- self.create_variable()
+ self.create_variables()
response = test_client.post("/public/variables/", json=body)
assert response.status_code == 201
assert response.json() == expected_response