This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 0cb875b7ec AIP-44 Migrate MetastoreBackend to Internal API (#33829)
0cb875b7ec is described below

commit 0cb875b7ec1cebb101866581166cd7b97047f941
Author: mhenc <[email protected]>
AuthorDate: Tue Aug 29 12:24:10 2023 +0200

    AIP-44 Migrate MetastoreBackend to Internal API (#33829)
    
    
    
    ---------
    
    Co-authored-by: Jarek Potiuk <[email protected]>
---
 airflow/api_internal/endpoints/rpc_api_endpoint.py |  3 +
 airflow/models/connection.py                       |  4 ++
 airflow/secrets/metastore.py                       | 23 +++++--
 airflow/serialization/enums.py                     |  1 +
 airflow/serialization/serialized_objects.py        |  4 ++
 .../endpoints/test_rpc_api_endpoint.py             | 70 ++++++++++------------
 6 files changed, 60 insertions(+), 45 deletions(-)

diff --git a/airflow/api_internal/endpoints/rpc_api_endpoint.py 
b/airflow/api_internal/endpoints/rpc_api_endpoint.py
index 22a4f7cb64..f241e1149b 100644
--- a/airflow/api_internal/endpoints/rpc_api_endpoint.py
+++ b/airflow/api_internal/endpoints/rpc_api_endpoint.py
@@ -39,6 +39,7 @@ def _initialize_map() -> dict[str, Callable]:
     from airflow.models import Trigger, Variable, XCom
     from airflow.models.dag import DagModel
     from airflow.models.dagwarning import DagWarning
+    from airflow.secrets.metastore import MetastoreBackend
 
     functions: list[Callable] = [
         DagFileProcessor.update_import_errors,
@@ -48,6 +49,8 @@ def _initialize_map() -> dict[str, Callable]:
         DagModel.get_paused_dag_ids,
         DagFileProcessorManager.clear_nonexistent_import_errors,
         DagWarning.purge_inactive_dag_warnings,
+        MetastoreBackend._fetch_connection,
+        MetastoreBackend._fetch_variable,
         XCom.get_value,
         XCom.get_one,
         XCom.get_many,
diff --git a/airflow/models/connection.py b/airflow/models/connection.py
index e0ba22edd7..f67d7800e1 100644
--- a/airflow/models/connection.py
+++ b/airflow/models/connection.py
@@ -21,6 +21,7 @@ import json
 import logging
 import warnings
 from json import JSONDecodeError
+from typing import Any
 from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
 
 from sqlalchemy import Boolean, Column, Integer, String, Text
@@ -476,6 +477,9 @@ class Connection(Base, LoggingMixin):
 
         raise AirflowNotFoundException(f"The conn_id `{conn_id}` isn't 
defined")
 
+    def to_dict(self) -> dict[str, Any]:
+        return {"conn_id": self.conn_id, "description": self.description, 
"uri": self.get_uri()}
+
     @classmethod
     def from_json(cls, value, conn_id=None) -> Connection:
         kwargs = json.loads(value)
diff --git a/airflow/secrets/metastore.py b/airflow/secrets/metastore.py
index ec54d432d0..481b7aaf3a 100644
--- a/airflow/secrets/metastore.py
+++ b/airflow/secrets/metastore.py
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING
 
 from sqlalchemy import select
 
+from airflow.api_internal.internal_api_call import internal_api_call
 from airflow.exceptions import RemovedInAirflow3Warning
 from airflow.secrets import BaseSecretsBackend
 from airflow.utils.session import NEW_SESSION, provide_session
@@ -38,11 +39,7 @@ class MetastoreBackend(BaseSecretsBackend):
 
     @provide_session
     def get_connection(self, conn_id: str, session: Session = NEW_SESSION) -> 
Connection | None:
-        from airflow.models.connection import Connection
-
-        conn = session.scalar(select(Connection).where(Connection.conn_id == 
conn_id).limit(1))
-        session.expunge_all()
-        return conn
+        return MetastoreBackend._fetch_connection(conn_id, session=session)
 
     @provide_session
     def get_connections(self, conn_id: str, session: Session = NEW_SESSION) -> 
list[Connection]:
@@ -65,6 +62,22 @@ class MetastoreBackend(BaseSecretsBackend):
         :param key: Variable Key
         :return: Variable Value
         """
+        return MetastoreBackend._fetch_variable(key=key, session=session)
+
+    @staticmethod
+    @internal_api_call
+    @provide_session
+    def _fetch_connection(conn_id: str, session: Session = NEW_SESSION) -> 
Connection | None:
+        from airflow.models.connection import Connection
+
+        conn = session.scalar(select(Connection).where(Connection.conn_id == 
conn_id).limit(1))
+        session.expunge_all()
+        return conn
+
+    @staticmethod
+    @internal_api_call
+    @provide_session
+    def _fetch_variable(key: str, session: Session = NEW_SESSION) -> str | 
None:
         from airflow.models.variable import Variable
 
         var_value = session.scalar(select(Variable).where(Variable.key == 
key).limit(1))
diff --git a/airflow/serialization/enums.py b/airflow/serialization/enums.py
index 0b1c0ca009..7452b1d36c 100644
--- a/airflow/serialization/enums.py
+++ b/airflow/serialization/enums.py
@@ -55,4 +55,5 @@ class DagAttributeTypes(str, Enum):
     TASK_INSTANCE = "task_instance"
     DAG_RUN = "dag_run"
     DATA_SET = "data_set"
+    CONNECTION = "connection"
     ARG_NOT_SET = "arg_not_set"
diff --git a/airflow/serialization/serialized_objects.py 
b/airflow/serialization/serialized_objects.py
index a63843e971..230f4aca18 100644
--- a/airflow/serialization/serialized_objects.py
+++ b/airflow/serialization/serialized_objects.py
@@ -487,6 +487,8 @@ class BaseSerialization:
                 cls.serialize(var.__dict__, strict=strict, 
use_pydantic_models=use_pydantic_models),
                 type_=DAT.SIMPLE_TASK_INSTANCE,
             )
+        elif isinstance(var, Connection):
+            return cls._encode(var.to_dict(), type_=DAT.CONNECTION)
         elif use_pydantic_models and _ENABLE_AIP_44:
 
             def _pydantic_model_dump(model_cls: type[BaseModel], var: Any) -> 
dict[str, Any]:
@@ -571,6 +573,8 @@ class BaseSerialization:
             return Dataset(**var)
         elif type_ == DAT.SIMPLE_TASK_INSTANCE:
             return SimpleTaskInstance(**cls.deserialize(var))
+        elif type_ == DAT.CONNECTION:
+            return Connection(**var)
         elif use_pydantic_models and _ENABLE_AIP_44:
             if type_ == DAT.BASE_JOB:
                 return JobPydantic.parse_obj(var)
diff --git a/tests/api_internal/endpoints/test_rpc_api_endpoint.py 
b/tests/api_internal/endpoints/test_rpc_api_endpoint.py
index bd7a7dc189..d72448324e 100644
--- a/tests/api_internal/endpoints/test_rpc_api_endpoint.py
+++ b/tests/api_internal/endpoints/test_rpc_api_endpoint.py
@@ -23,6 +23,7 @@ from unittest import mock
 import pytest
 from flask import Flask
 
+from airflow.models.connection import Connection
 from airflow.models.taskinstance import TaskInstance
 from airflow.operators.empty import EmptyOperator
 from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic
@@ -54,6 +55,10 @@ def minimal_app_for_internal_api() -> Flask:
     return factory()
 
 
+def equals(a, b) -> bool:
+    return a == b
+
+
 @pytest.mark.skipif(not _ENABLE_AIP_44, reason="AIP-44 is disabled")
 class TestRpcApiEndpoint:
     @pytest.fixture(autouse=True)
@@ -71,65 +76,50 @@ class TestRpcApiEndpoint:
             yield mock_initialize_map
 
     @pytest.mark.parametrize(
-        "input_data, method_result, method_params, expected_mock, 
expected_code",
+        "input_params, method_result, result_cmp_func, method_params",
         [
+            ("", None, equals, {}),
+            ("", "test_me", equals, {}),
             (
-                {"jsonrpc": "2.0", "method": TEST_METHOD_NAME, "params": ""},
-                "test_me",
-                {},
-                mock_test_method,
-                200,
+                json.dumps(BaseSerialization.serialize({"dag_id": 15, 
"task_id": "fake-task"})),
+                ("dag_id_15", "fake-task", 1),
+                equals,
+                {"dag_id": 15, "task_id": "fake-task"},
             ),
             (
-                {"jsonrpc": "2.0", "method": TEST_METHOD_NAME, "params": ""},
-                None,
+                "",
+                TaskInstance(task=EmptyOperator(task_id="task"), 
run_id="run_id", state=State.RUNNING),
+                lambda a, b: a == TaskInstancePydantic.from_orm(b),
                 {},
-                mock_test_method,
-                200,
             ),
             (
-                {
-                    "jsonrpc": "2.0",
-                    "method": TEST_METHOD_NAME,
-                    "params": 
json.dumps(BaseSerialization.serialize({"dag_id": 15, "task_id": "fake-task"})),
-                },
-                ("dag_id_15", "fake-task", 1),
-                {"dag_id": 15, "task_id": "fake-task"},
-                mock_test_method,
-                200,
+                "",
+                Connection(conn_id="test_conn", conn_type="http", host="", 
password=""),
+                lambda a, b: a.get_uri() == b.get_uri() and a.conn_id == 
b.conn_id,
+                {},
             ),
         ],
     )
-    def test_method(self, input_data, method_result, method_params, 
expected_mock, expected_code):
+    def test_method(self, input_params, method_result, result_cmp_func, 
method_params):
         if method_result:
-            expected_mock.return_value = method_result
+            mock_test_method.return_value = method_result
 
+        input_data = {
+            "jsonrpc": "2.0",
+            "method": TEST_METHOD_NAME,
+            "params": input_params,
+        }
         response = self.client.post(
             "/internal_api/v1/rpcapi",
             headers={"Content-Type": "application/json"},
             data=json.dumps(input_data),
         )
-        assert response.status_code == expected_code
+        assert response.status_code == 200
         if method_result:
-            response_data = 
BaseSerialization.deserialize(json.loads(response.data))
-            assert response_data == method_result
-
-        expected_mock.assert_called_once_with(**method_params)
+            response_data = 
BaseSerialization.deserialize(json.loads(response.data), 
use_pydantic_models=True)
+            assert result_cmp_func(response_data, method_result)
 
-    def test_method_with_pydantic_serialized_object(self):
-        ti = TaskInstance(task=EmptyOperator(task_id="task"), run_id="run_id", 
state=State.RUNNING)
-        mock_test_method.return_value = ti
-
-        response = self.client.post(
-            "/internal_api/v1/rpcapi",
-            headers={"Content-Type": "application/json"},
-            data=json.dumps({"jsonrpc": "2.0", "method": TEST_METHOD_NAME, 
"params": ""}),
-        )
-        assert response.status_code == 200
-        print(response.data)
-        response_data = 
BaseSerialization.deserialize(json.loads(response.data), 
use_pydantic_models=True)
-        expected_data = TaskInstancePydantic.from_orm(ti)
-        assert response_data == expected_data
+        mock_test_method.assert_called_once_with(**method_params)
 
     def test_method_with_exception(self):
         mock_test_method.side_effect = ValueError("Error!!!")

Reply via email to