Copilot commented on code in PR #62343: URL: https://github.com/apache/airflow/pull/62343#discussion_r3066488476
########## airflow-core/tests/unit/models/test_connection_test.py: ########## @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.models.connection import Connection +from airflow.models.connection_test import ( + ConnectionTestRequest, + ConnectionTestState, + run_connection_test, +) + +from tests_common.test_utils.db import clear_db_connection_tests, clear_db_connections + +pytestmark = pytest.mark.db_test + + +class TestConnectionTestRequestModel: + def test_token_is_generated(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + assert ct.token is not None + assert len(ct.token) > 0 + + def test_initial_state_is_pending(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + assert ct.state == ConnectionTestState.PENDING + + def test_tokens_are_unique(self): + ct1 = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + ct2 = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + assert ct1.token != ct2.token + + def test_repr(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + r = repr(ct) + assert "test_conn" in r + assert "pending" in r + + def test_executor_parameter(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres", executor="my_executor") + assert ct.executor == "my_executor" + + def test_executor_defaults_to_none(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + assert ct.executor is None + + def test_queue_parameter(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres", queue="my_queue") + assert ct.queue == "my_queue" + + def test_queue_defaults_to_none(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + assert ct.queue is None + + def test_connection_fields_stored(self): + ct = ConnectionTestRequest( + connection_id="test_conn", + conn_type="postgres", + host="db.example.com", + login="user", + password="secret", + schema="mydb", + port=5432, + extra='{"key": "value"}', + ) + assert ct.conn_type == "postgres" + assert ct.host == "db.example.com" + assert ct.login == "user" + assert ct.password == "secret" + assert ct.schema == "mydb" + assert ct.port == 5432 + assert ct.extra == '{"key": "value"}' + + def test_password_is_encrypted(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres", password="secret") + assert ct._password is not None + assert ct._password != "secret" + assert ct.password == "secret" + + def test_extra_is_encrypted(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres", extra='{"key": "val"}') + assert ct._extra is not None + assert ct._extra != '{"key": "val"}' + assert ct.extra == '{"key": "val"}' + + def test_null_password_and_extra(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="http") + assert ct._password is None + assert ct._extra is None + + def test_commit_on_success_default(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres") + assert ct.commit_on_success is False + + def test_commit_on_success_true(self): + ct = ConnectionTestRequest(connection_id="test_conn", conn_type="postgres", commit_on_success=True) + assert ct.commit_on_success is True + + +class TestToConnection: + def test_to_connection_returns_transient_connection(self): + ct = ConnectionTestRequest( + connection_id="test_conn", + conn_type="postgres", + host="db.example.com", + login="user", + password="secret", + schema="mydb", + port=5432, + extra='{"key": "value"}', + ) + conn = ct.to_connection() + assert isinstance(conn, Connection) + assert conn.conn_id == "test_conn" + assert conn.conn_type == "postgres" + assert conn.host == "db.example.com" + assert conn.login == "user" + assert conn.password == "secret" + assert conn.schema == "mydb" + assert conn.port == 5432 + assert conn.extra == '{"key": "value"}' + + +class TestCommitToConnectionTable: + @pytest.fixture(autouse=True) + def setup_teardown(self): + clear_db_connections(add_default_connections_back=False) + clear_db_connection_tests() + yield + clear_db_connections(add_default_connections_back=False) + clear_db_connection_tests() + + def test_creates_new_connection(self, session): + ct = ConnectionTestRequest( + connection_id="new_conn", + conn_type="postgres", + host="db.example.com", + login="user", + password="secret", + schema="mydb", + port=5432, + ) + session.add(ct) + session.flush() + + ct.commit_to_connection_table(session=session) + session.flush() + + from sqlalchemy import select Review Comment: Avoid imports inside test functions; move `from sqlalchemy import select` to the module import section so the test file has a consistent import style and avoids per-test import side effects. ########## airflow-core/src/airflow/config_templates/config.yml: ########## @@ -2567,6 +2567,33 @@ scheduler: type: float example: ~ default: "120.0" + connection_test_timeout: + description: | + Maximum number of seconds an async connection test is allowed to run + before it is considered timed out. The scheduler reaper uses this value + plus a grace period to mark stale tests as failed. + version_added: 3.3.0 + type: integer + example: ~ + default: "60" + max_connection_test_concurrency: + description: | + Maximum number of connection tests that can be active (QUEUED + RUNNING) + at the same time. Excess tests will remain in PENDING state until slots + become available. + version_added: 3.3.0 + type: integer + example: ~ + default: "4" Review Comment: PR description/config section mentions `[core] connection_test_timeout` and `[core] max_connection_test_concurrency`, but the actual settings added here are under `[scheduler]` (and the scheduler code reads them from `[scheduler]`). Please align the documented config location (or move the settings) so operators aren’t misled about which section to configure. ########## airflow-core/src/airflow/models/connection_test.py: ########## @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import secrets +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING +from uuid import UUID + +import structlog +import uuid6 +from sqlalchemy import Boolean, Index, Integer, String, Text, Uuid, select, text +from sqlalchemy.orm import Mapped, mapped_column + +from airflow._shared.timezones import timezone +from airflow.models.base import Base +from airflow.models.connection import Connection +from airflow.models.crypto import FernetFieldsMixin +from airflow.utils.sqlalchemy import UtcDateTime + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + +log = structlog.get_logger(__name__) + + +class ConnectionTestState(str, Enum): + """All possible states of a connection test.""" + + PENDING = "pending" + QUEUED = "queued" + RUNNING = "running" + SUCCESS = "success" + FAILED = "failed" + + def __str__(self) -> str: + return self.value + + +ACTIVE_STATES = frozenset( + (ConnectionTestState.PENDING, ConnectionTestState.QUEUED, ConnectionTestState.RUNNING) +) +DISPATCHED_STATES = frozenset((ConnectionTestState.QUEUED, ConnectionTestState.RUNNING)) +TERMINAL_STATES = frozenset((ConnectionTestState.SUCCESS, ConnectionTestState.FAILED)) + + +class ConnectionTestRequest(Base, FernetFieldsMixin): + """ + Tracks an async connection test request dispatched to a worker. + + Stores the full connection details so the worker reads from this table + instead of the real ``connection`` table. The real ``connection`` table + is only modified if the test succeeds and ``commit_on_success`` is True. + """ + + __tablename__ = "connection_test_request" + + id: Mapped[UUID] = mapped_column(Uuid(), primary_key=True, default=uuid6.uuid7) + token: Mapped[str] = mapped_column(String(64), nullable=False, unique=True) + connection_id: Mapped[str] = mapped_column(String(250), nullable=False) + state: Mapped[str] = mapped_column(String(20), nullable=False, default=ConnectionTestState.PENDING) + result_message: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(UtcDateTime, default=timezone.utcnow, nullable=False) + updated_at: Mapped[datetime] = mapped_column( + UtcDateTime, default=timezone.utcnow, onupdate=timezone.utcnow, nullable=False + ) + executor: Mapped[str | None] = mapped_column(String(256), nullable=True) + queue: Mapped[str | None] = mapped_column(String(256), nullable=True) + + # Connection fields — password and extra are Fernet-encrypted via FernetFieldsMixin. + conn_type: Mapped[str] = mapped_column(String(500), nullable=False) + host: Mapped[str | None] = mapped_column(String(500), nullable=True) + login: Mapped[str | None] = mapped_column(Text, nullable=True) + schema: Mapped[str | None] = mapped_column("schema", String(500), nullable=True) + port: Mapped[int | None] = mapped_column(Integer, nullable=True) + commit_on_success: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0" + ) + + __table_args__ = ( + Index("idx_connection_test_request_state_created_at", state, created_at), + # since mysql lacks filtered/partial indices, this creates a + # duplicate index on mysql. Not the end of the world + Index( + "idx_connection_test_request_active_conn", + "connection_id", + unique=True, + postgresql_where=text("state IN ('pending', 'queued', 'running')"), + sqlite_where=text("state IN ('pending', 'queued', 'running')"), + ), Review Comment: The `idx_connection_test_request_active_conn` index is declared `unique=True` with `postgresql_where`/`sqlite_where`. On MySQL (which doesn't support partial indexes), this will compile as an unconditional UNIQUE index on `connection_id`, preventing *any* subsequent connection tests for the same `connection_id` even after earlier tests reach terminal states. Consider making this non-unique on MySQL (conditional DDL in the migration) or using a MySQL-compatible pattern (e.g., generated/nullable “active_connection_id” column with a unique index) to enforce uniqueness only for active states. ########## airflow-core/src/airflow/migrations/versions/0111_3_3_0_add_connection_test_table.py: ########## @@ -0,0 +1,88 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Add connection_test_request table for async connection testing. + +Revision ID: a7e6d4c3b2f1 +Revises: a4c2d171ae18 +Create Date: 2026-02-22 00:00:00.000000 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +from airflow.utils.sqlalchemy import UtcDateTime + +# revision identifiers, used by Alembic. +revision = "a7e6d4c3b2f1" +down_revision = "a4c2d171ae18" +branch_labels = None +depends_on = None +airflow_version = "3.3.0" + + +def upgrade(): + """Create connection_test_request table.""" + op.create_table( + "connection_test_request", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("token", sa.String(64), nullable=False), + sa.Column("connection_id", sa.String(250), nullable=False), + sa.Column("state", sa.String(20), nullable=False), + sa.Column("result_message", sa.Text(), nullable=True), + sa.Column("created_at", UtcDateTime(timezone=True), nullable=False), + sa.Column("updated_at", UtcDateTime(timezone=True), nullable=False), + sa.Column("executor", sa.String(256), nullable=True), + sa.Column("queue", sa.String(256), nullable=True), + sa.Column("conn_type", sa.String(500), nullable=False), + sa.Column("host", sa.String(500), nullable=True), + sa.Column("login", sa.Text(), nullable=True), + sa.Column("password", sa.Text(), nullable=True), + sa.Column("schema", sa.String(500), nullable=True), + sa.Column("port", sa.Integer(), nullable=True), + sa.Column("extra", sa.Text(), nullable=True), + sa.Column("commit_on_success", sa.Boolean(), nullable=False, server_default="0"), + sa.PrimaryKeyConstraint("id", name=op.f("connection_test_request_pkey")), + sa.UniqueConstraint("token", name=op.f("connection_test_request_token_uq")), + ) + op.create_index( + op.f("idx_connection_test_request_state_created_at"), + "connection_test_request", + ["state", "created_at"], + ) + # since mysql lacks filtered/partial indices, this creates a + # duplicate index on mysql + op.create_index( + op.f("idx_connection_test_request_active_conn"), + "connection_test_request", + ["connection_id"], + unique=True, + postgresql_where=sa.text("state IN ('pending', 'queued', 'running')"), + sqlite_where=sa.text("state IN ('pending', 'queued', 'running')"), + ) + + Review Comment: This migration creates `idx_connection_test_request_active_conn` as `unique=True` with only `postgresql_where`/`sqlite_where`. On MySQL, that becomes an unconditional UNIQUE index on `connection_id`, which will block creating a second test request for the same connection after the first one completes. Please adjust the migration to avoid an unconditional unique index on MySQL (dialect-conditional index creation or a MySQL-compatible partial-uniqueness workaround). ```suggestion dialect_name = op.get_bind().dialect.name if dialect_name in {"postgresql", "sqlite"}: op.create_index( op.f("idx_connection_test_request_active_conn"), "connection_test_request", ["connection_id"], unique=True, postgresql_where=sa.text("state IN ('pending', 'queued', 'running')"), sqlite_where=sa.text("state IN ('pending', 'queued', 'running')"), ) else: # MySQL does not support filtered/partial indexes. Create a plain # non-unique index instead of an unconditional unique index. op.create_index( op.f("idx_connection_test_request_active_conn"), "connection_test_request", ["connection_id"], unique=False, ) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
