This is an automated email from the ASF dual-hosted git repository.
ashb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6a83e5e863c Use a separate session in MetastoreBackend when no session
is passed (#76547)
6a83e5e863c is described below
commit 6a83e5e863c0cc331d15aa93afb651dd71f70481
Author: Adam Postek <[email protected]>
AuthorDate: Wed Nov 18 09:06:44 2026 +0000
Use a separate session in MetastoreBackend when no session is passed
(#76547)
Signed-off-by: Adam <[email protected]>
---
airflow-core/src/airflow/secrets/metastore.py | 16 +++++++----
.../tests/unit/always/test_secrets_metastore.py | 31 ++++++++++++++++++++++
airflow-core/tests/unit/models/test_variable.py | 26 +++++++++---------
3 files changed, 55 insertions(+), 18 deletions(-)
diff --git a/airflow-core/src/airflow/secrets/metastore.py
b/airflow-core/src/airflow/secrets/metastore.py
index 78e124b423a..f7bb445f8c3 100644
--- a/airflow-core/src/airflow/secrets/metastore.py
+++ b/airflow-core/src/airflow/secrets/metastore.py
@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING
from sqlalchemy import or_, select
from airflow.secrets import BaseSecretsBackend
-from airflow.utils.session import NEW_SESSION, provide_session
+from airflow.utils.session import create_session
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -35,9 +35,8 @@ if TYPE_CHECKING:
class MetastoreBackend(BaseSecretsBackend):
"""Retrieves Connection object and Variable from airflow metastore
database."""
- @provide_session
def get_connection(
- self, conn_id: str, team_name: str | None = None, *, session: Session
= NEW_SESSION
+ self, conn_id: str, team_name: str | None = None, *, session: Session
| None = None
) -> Connection | None:
"""
Get Airflow Connection from Metadata DB.
@@ -47,6 +46,10 @@ class MetastoreBackend(BaseSecretsBackend):
:param session: SQLAlchemy Session
:return: Connection Object
"""
+ if session is None:
+ with create_session(scoped=False) as session:
+ return self.get_connection(conn_id, team_name, session=session)
+
from airflow.models import Connection
conn = session.scalar(
@@ -61,9 +64,8 @@ class MetastoreBackend(BaseSecretsBackend):
session.expunge(conn)
return conn
- @provide_session
def get_variable(
- self, key: str, team_name: str | None = None, *, session: Session =
NEW_SESSION
+ self, key: str, team_name: str | None = None, *, session: Session |
None = None
) -> str | None:
"""
Get Airflow Variable from Metadata DB.
@@ -73,6 +75,10 @@ class MetastoreBackend(BaseSecretsBackend):
:param session: SQLAlchemy Session
:return: Variable Value
"""
+ if session is None:
+ with create_session(scoped=False) as session:
+ return self.get_variable(key, team_name, session=session)
+
from airflow.models import Variable
var_value = session.scalar(
diff --git a/airflow-core/tests/unit/always/test_secrets_metastore.py
b/airflow-core/tests/unit/always/test_secrets_metastore.py
index d13419563f7..bc0735de6af 100644
--- a/airflow-core/tests/unit/always/test_secrets_metastore.py
+++ b/airflow-core/tests/unit/always/test_secrets_metastore.py
@@ -18,7 +18,9 @@
from __future__ import annotations
import pytest
+from sqlalchemy import select
+from airflow import settings
from airflow.models.connection import Connection
from airflow.models.variable import Variable
from airflow.secrets.metastore import MetastoreBackend
@@ -108,3 +110,32 @@ class TestMetastoreBackendSessionSafety:
# Attributes should still be accessible
assert conn.conn_id == "test_conn"
assert conn.host == "localhost"
+
+ @pytest.mark.parametrize("exists", [True, False], ids=["found",
"not_found"])
+ @pytest.mark.parametrize("lookup", ["get_connection", "get_variable"])
+ def
test_lookup_without_session_does_not_commit_or_close_caller_session(self,
lookup, exists):
+ """A lookup without a session must not commit or close the caller's
scoped session."""
+ with create_session() as session:
+ session.add(Connection(conn_id="loaded_conn", conn_type="http"))
+ if exists:
+ with create_session() as session:
+ session.add(Connection(conn_id="target", conn_type="mysql"))
+ Variable.set(key="target", value="value")
+
+ session = settings.Session()
+ try:
+ loaded =
session.scalar(select(Connection).where(Connection.conn_id == "loaded_conn"))
+ pending = Connection(conn_id="pending_conn", conn_type="http")
+ session.add(pending)
+
+ result = getattr(MetastoreBackend(), lookup)("target")
+
+ assert (result is not None) == exists
+ assert loaded in session
+ assert pending in session.new
+ session.rollback()
+ finally:
+ settings.Session.remove()
+
+ with create_session() as session:
+ assert session.scalar(select(Connection).where(Connection.conn_id
== "pending_conn")) is None
diff --git a/airflow-core/tests/unit/models/test_variable.py
b/airflow-core/tests/unit/models/test_variable.py
index 57742aff4fa..6c1ea3c5301 100644
--- a/airflow-core/tests/unit/models/test_variable.py
+++ b/airflow-core/tests/unit/models/test_variable.py
@@ -157,13 +157,13 @@ class TestVariable:
@conf_vars({("core", "multi_team"): "True"})
def test_get_variable_with_team(self, testing_team, session):
Variable.set(key="key", value="value", team_name=testing_team.name,
session=session)
- result = Variable.get(key="key", team_name=testing_team.name)
+ result = Variable.get(key="key", team_name=testing_team.name,
session=session)
assert result == "value"
@conf_vars({("core", "multi_team"): "True"})
def test_get_global_variable_with_team(self, testing_team, session):
Variable.set(key="key", value="value", session=session)
- result = Variable.get(key="key", team_name=testing_team.name)
+ result = Variable.get(key="key", team_name=testing_team.name,
session=session)
assert result == "value"
@conf_vars({("core", "multi_team"): "True"})
@@ -192,7 +192,7 @@ class TestVariable:
# invalidate the cache to re-evaluate value
SecretCache.invalidate_variable("key")
# now that env var is not here anymore, we see the value we set before.
- assert Variable.get("key") == "new-db-value"
+ assert Variable.get("key", session=session) == "new-db-value"
assert caplog.messages[0] == (
"The variable key is defined in the EnvironmentVariablesBackend
secrets backend, "
@@ -379,9 +379,9 @@ class TestVariable:
def test_variable_update(self, session):
Variable.set(key="test_key", value="value1", session=session)
- assert Variable.get(key="test_key") == "value1"
+ assert Variable.get(key="test_key", session=session) == "value1"
Variable.update(key="test_key", value="value2", session=session)
- assert Variable.get("test_key") == "value2"
+ assert Variable.get("test_key", session=session) == "value2"
def test_variable_update_fails_on_non_metastore_variable(self, session):
with mock.patch.dict("os.environ", AIRFLOW_VAR_KEY="env-value"):
@@ -390,7 +390,7 @@ class TestVariable:
def test_variable_update_preserves_description(self, session):
Variable.set(key="key", value="value", description="a test variable",
session=session)
- assert Variable.get("key") == "value"
+ assert Variable.get("key", session=session) == "value"
Variable.update("key", "value2")
test_var = session.scalar(select(Variable).where(Variable.key ==
"key"))
assert test_var.val == "value2"
@@ -400,13 +400,13 @@ class TestVariable:
def test_variable_update_with_team(self, testing_team, session):
Variable.set(key="test_key", value="value1",
team_name=testing_team.name, session=session)
Variable.update(key="test_key", value="value2",
team_name=testing_team.name, session=session)
- assert Variable.get("test_key", team_name=testing_team.name) ==
"value2"
+ assert Variable.get("test_key", team_name=testing_team.name,
session=session) == "value2"
@conf_vars({("core", "multi_team"): "True"})
def test_variable_update_with_team_global(self, testing_team, session):
Variable.set(key="test_key", value="value1", session=session)
Variable.update(key="test_key", value="value2",
team_name=testing_team.name, session=session)
- assert Variable.get("test_key", team_name=testing_team.name) ==
"value2"
+ assert Variable.get("test_key", team_name=testing_team.name,
session=session) == "value2"
@conf_vars({("core", "multi_team"): "True"})
def test_variable_update_with_wrong_team(self, testing_team, session):
@@ -436,7 +436,7 @@ class TestVariable:
test_key = "test_key"
Variable.set(key=test_key, value=test_value, session=session)
Variable.set(key=test_key, value="", session=session)
- assert Variable.get("test_key") == ""
+ assert Variable.get("test_key", session=session) == ""
def test_get_non_existing_var_should_return_default(self):
default_value = "some default val"
@@ -491,12 +491,12 @@ class TestVariable:
# Set the variable
Variable.set(key=key, value=value, session=session)
- assert value == Variable.get(key)
+ assert value == Variable.get(key, session=session)
# Delete the variable
Variable.delete(key=key, session=session)
with pytest.raises(KeyError):
- Variable.get(key)
+ Variable.get(key, session=session)
@conf_vars({("core", "multi_team"): "True"})
def test_variable_delete_with_team(self, testing_team, session):
@@ -555,7 +555,7 @@ class TestVariable:
Variable.set(key="k", value="v2", team_name=testing_team.name,
session=session)
- assert Variable.get("k") == "v2"
+ assert Variable.get("k", session=session) == "v2"
@mock.patch("airflow.models.variable.ensure_secrets_loaded")
def test_caching_caches(self, mock_ensure_secrets: mock.Mock):
@@ -581,7 +581,7 @@ class TestVariable:
# setting a new value invalidates the cache
Variable.set(key="key", value="new_value", session=session)
- c = Variable.get("key") # cache should not be used
+ c = Variable.get("key", session=session) # cache should not be used
assert c != b