This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 258a0b10498 Unify `queue` parameter on `MessageQueueTrigger` with
trigger queues (#77046)
258a0b10498 is described below
commit 258a0b10498ef4b96c54f07904563c8422c32c78
Author: Arif Rahman <[email protected]>
AuthorDate: Wed Nov 18 05:34:59 2026 +0700
Unify `queue` parameter on `MessageQueueTrigger` with trigger queues
(#77046)
---
.../tests_common/test_utils/common_msg_queue.py | 2 +-
providers/common/messaging/docs/changelog.rst | 8 +++
providers/common/messaging/docs/triggers.rst | 5 ++
.../common/messaging/triggers/msg_queue.py | 54 ++++++++++-----
.../common/messaging/triggers/test_msg_queue.py | 78 +++++++++++++++++-----
5 files changed, 113 insertions(+), 34 deletions(-)
diff --git a/devel-common/src/tests_common/test_utils/common_msg_queue.py
b/devel-common/src/tests_common/test_utils/common_msg_queue.py
index c0b19eec7d1..05de5fd2411 100644
--- a/devel-common/src/tests_common/test_utils/common_msg_queue.py
+++ b/devel-common/src/tests_common/test_utils/common_msg_queue.py
@@ -29,7 +29,7 @@ def collect_queue_param_deprecation_warning():
"""Collect deprecation warnings for queue parameter."""
with pytest.warns(
AirflowProviderDeprecationWarning,
- match="The `queue` parameter is deprecated and will be removed in
future versions. Use the `scheme` parameter instead and pass configuration as
keyword arguments to `MessageQueueTrigger`.",
+ match="Passing the broker queue URI as `queue` is deprecated",
):
yield
diff --git a/providers/common/messaging/docs/changelog.rst
b/providers/common/messaging/docs/changelog.rst
index 22a5acce5dd..fc8548c2a0f 100644
--- a/providers/common/messaging/docs/changelog.rst
+++ b/providers/common/messaging/docs/changelog.rst
@@ -25,6 +25,14 @@
Changelog
---------
+.. note::
+
+ In ``MessageQueueTrigger``, the ``queue`` parameter now assigns the
trigger to a trigger queue
+ when used with ``scheme``, aligning with ``BaseEventTrigger(queue=...)``.
+ The ``trigger_queue`` parameter introduced in 2.1.0 is deprecated in favor
of ``queue``.
+ Passing a broker queue URI as ``queue`` is deprecated; use ``scheme``
instead and pass
+ broker configuration as keyword arguments.
+
2.1.2
.....
diff --git a/providers/common/messaging/docs/triggers.rst
b/providers/common/messaging/docs/triggers.rst
index 35c88f66656..847cf224ef1 100644
--- a/providers/common/messaging/docs/triggers.rst
+++ b/providers/common/messaging/docs/triggers.rst
@@ -29,6 +29,11 @@ queue. Mandatory parameters of the trigger are:
- ``scheme`` - the queue scheme (e.g., 'kafka', 'redis+pubsub', 'sqs') you are
using
+Optional parameters include:
+
+- ``queue`` - assign this trigger to a specific trigger queue (see
+ :ref:`config:triggerer__queues_enabled` and the ``--queues`` option of
``airflow triggerer``)
+
Additional parameters can be provided depending on the queue provider.
Connections needs to be provided with the relevant
default connection ID, for example, when connecting to a queue in AWS SQS, the
connection ID should be
``aws_default``.
diff --git
a/providers/common/messaging/src/airflow/providers/common/messaging/triggers/msg_queue.py
b/providers/common/messaging/src/airflow/providers/common/messaging/triggers/msg_queue.py
index d69407e888c..19b2decf3b3 100644
---
a/providers/common/messaging/src/airflow/providers/common/messaging/triggers/msg_queue.py
+++
b/providers/common/messaging/src/airflow/providers/common/messaging/triggers/msg_queue.py
@@ -50,12 +50,13 @@ class MessageQueueTrigger(BaseEventTrigger):
This makes it easy to switch providers without modifying the trigger.
:param scheme: The queue scheme (e.g., 'kafka', 'redis+pubsub', 'sqs').
Used for provider matching.
- :param queue: **Deprecated** The queue identifier (URI format). If
provided, this takes precedence over scheme parameter.
- This parameter is deprecated and will be removed in future versions.
Use the 'scheme' parameter instead.
- :param trigger_queue: Assign this trigger to a specific trigger queue (see
+ :param queue: Assign this trigger to a specific trigger queue (see
:ref:`config:triggerer__queues_enabled` and the ``--queues`` option of
``airflow triggerer``).
- Named differently from the ``queue`` parameter above, which is the
deprecated broker queue URI
- and cannot be repurposed for triggerer routing without breaking
existing callers.
+ If ``scheme`` is not provided, or the value contains ``://`` or
matches a registered broker
+ queue pattern, this parameter is treated as the deprecated broker
queue URI for backward
+ compatibility.
+ :param trigger_queue: **Deprecated** Assign this trigger to a specific
trigger queue.
+ Deprecated alias for ``queue``. Use the ``scheme`` parameter together
with ``queue`` instead.
.. seealso::
For more information on how to use this trigger, take a look at the
guide:
@@ -68,32 +69,49 @@ class MessageQueueTrigger(BaseEventTrigger):
def __init__(
self,
*,
- queue: str | None = None,
scheme: str | None = None,
+ queue: str | None = None,
trigger_queue: str | None = None,
**kwargs: Any,
) -> None:
- # Stored under our own name rather than through
`BaseEventTrigger.__init__(queue=...)` so this
- # works regardless of the installed airflow-core version, see `queue`
property below.
- self._trigger_queue = trigger_queue
-
- if queue is None and scheme is None:
- raise ValueError("Either `queue` or `scheme` parameter must be
provided.")
-
- # For backward compatibility, queue takes precedence
- if queue is not None:
+ if queue is not None and (
+ scheme is None
+ or "://" in queue
+ or any(provider.queue_matches(queue) for provider in
MESSAGE_QUEUE_PROVIDERS)
+ ):
warnings.warn(
- "The `queue` parameter is deprecated and will be removed in
future versions. "
+ "Passing the broker queue URI as `queue` is deprecated and
will be removed in future versions. "
"Use the `scheme` parameter instead and pass configuration as
keyword arguments to `MessageQueueTrigger`.",
AirflowProviderDeprecationWarning,
stacklevel=2,
)
self.queue_uri = queue
- self.scheme = None
+ queue = None
else:
self.queue_uri = None
- self.scheme = scheme
+ if trigger_queue is not None:
+ warnings.warn(
+ "The `trigger_queue` parameter is deprecated and will be
removed in a future version. "
+ "Use the `scheme` parameter together with `queue` instead to
assign this trigger to a trigger queue.",
+ AirflowProviderDeprecationWarning,
+ stacklevel=2,
+ )
+
+ if queue is not None and trigger_queue is not None:
+ raise ValueError(
+ "Cannot provide both `queue` and deprecated `trigger_queue`.
For more details on "
+ "trigger queues, see: "
+
"https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/deferring.html#controlling-triggerer-host-assignment-per-trigger"
+ )
+
+ if self.queue_uri is None and scheme is None:
+ raise ValueError("The `scheme` parameter must be provided.")
+
+ # Stored under our own name rather than through
`BaseEventTrigger.__init__(queue=...)` so this
+ # works regardless of the installed airflow-core version, see `queue`
property below.
+ self._trigger_queue = queue if queue is not None else trigger_queue
+ self.scheme = scheme if self.queue_uri is None else None
self.kwargs = kwargs
@property
diff --git
a/providers/common/messaging/tests/unit/common/messaging/triggers/test_msg_queue.py
b/providers/common/messaging/tests/unit/common/messaging/triggers/test_msg_queue.py
index 7a84be4a94c..5dd2f878282 100644
---
a/providers/common/messaging/tests/unit/common/messaging/triggers/test_msg_queue.py
+++
b/providers/common/messaging/tests/unit/common/messaging/triggers/test_msg_queue.py
@@ -212,12 +212,43 @@ class TestMessageQueueTrigger:
class TestMessageQueueTriggerScheme:
"""Test cases for the new scheme parameter functionality."""
- @pytest.mark.usefixtures("collect_queue_param_deprecation_warning")
- def test_queue_takes_precedence_over_scheme(self):
- """Test that queue parameter takes precedence when both are
provided."""
- trigger = MessageQueueTrigger(queue=PROVIDER_1_QUEUE,
scheme=PROVIDER_2_SCHEME)
- assert trigger.queue_uri == PROVIDER_1_QUEUE
- assert trigger.scheme is None
+ @pytest.mark.parametrize(
+ ("providers", "queue", "scheme"),
+ [
+ pytest.param(
+ [
+ MockProvider(PROVIDER_1_NAME, PROVIDER_1_PATTERN,
scheme=PROVIDER_1_SCHEME),
+ MockProvider(PROVIDER_2_NAME, PROVIDER_2_PATTERN,
scheme=PROVIDER_2_SCHEME),
+ ],
+ PROVIDER_1_QUEUE,
+ PROVIDER_2_SCHEME,
+ id="uri_scheme_matched_by_provider",
+ ),
+ pytest.param(
+ [MockProvider("CustomProvider", pattern="custom-prefix-",
scheme="custom")],
+ "custom-prefix-abc",
+ "custom",
+ id="third_party_identifier_without_scheme",
+ ),
+ pytest.param(
+ [MockProvider("RedisProvider", pattern="nomatch://",
scheme="redis+pubsub")],
+ "redis+pubsub://localhost:6379/ch",
+ "redis+pubsub",
+ id="uri_scheme_unmatched_by_provider",
+ ),
+ ],
+ )
+ def test_legacy_queue_with_scheme_keeps_uri_matching(self, providers,
queue, scheme):
+ """A broker URI or identifier passed as `queue` next to `scheme` must
not become the trigger queue."""
+ with mock.patch(MESSAGE_QUEUE_PROVIDERS_PATH, providers):
+ with pytest.warns(
+ AirflowProviderDeprecationWarning,
+ match="Passing the broker queue URI as `queue` is deprecated",
+ ):
+ trigger = MessageQueueTrigger(queue=queue, scheme=scheme)
+ assert trigger.queue_uri == queue
+ assert trigger.scheme is None
+ assert trigger.queue is None
def test_scheme_only_initialization(self):
"""Test initialization with scheme parameter only."""
@@ -238,18 +269,35 @@ class TestMessageQueueTriggerScheme:
trigger = MessageQueueTrigger(scheme=PROVIDER_2_SCHEME)
assert trigger.queue is None
- @pytest.mark.usefixtures("collect_queue_param_deprecation_warning")
+ def test_queue_param_sets_trigger_queue_alongside_scheme(self):
+ """Test that the unified `queue` parameter sets the trigger queue when
used with scheme."""
+ trigger = MessageQueueTrigger(scheme=PROVIDER_2_SCHEME,
queue="my-trigger-queue")
+ assert trigger.scheme == PROVIDER_2_SCHEME
+ assert trigger.queue_uri is None
+ assert trigger.queue == "my-trigger-queue"
+
+
@pytest.mark.filterwarnings("ignore::airflow.exceptions.AirflowProviderDeprecationWarning")
def
test_trigger_queue_param_sets_trigger_queue_alongside_deprecated_queue(self):
- """The deprecated `queue` (broker URI) parameter claims the `queue`
keyword, so trigger
- queue routing must go through the distinct `trigger_queue` parameter
instead."""
+ """Test backward compatibility when passing a broker URI via `queue`
and trigger queue via deprecated `trigger_queue`."""
trigger = MessageQueueTrigger(queue=PROVIDER_1_QUEUE,
trigger_queue="my-trigger-queue")
assert trigger.queue_uri == PROVIDER_1_QUEUE
assert trigger.queue == "my-trigger-queue"
- def test_trigger_queue_param_sets_trigger_queue_alongside_scheme(self):
- trigger = MessageQueueTrigger(scheme=PROVIDER_2_SCHEME,
trigger_queue="my-trigger-queue")
+ def test_trigger_queue_param_deprecation_warning(self):
+ """Test that `trigger_queue` emits a deprecation warning in favor of
`queue`."""
+ with pytest.warns(
+ AirflowProviderDeprecationWarning,
+ match="The `trigger_queue` parameter is deprecated and will be
removed in a future version. Use the `scheme` parameter together with `queue`
instead",
+ ):
+ trigger = MessageQueueTrigger(scheme=PROVIDER_2_SCHEME,
trigger_queue="my-trigger-queue")
assert trigger.queue == "my-trigger-queue"
+
@pytest.mark.filterwarnings("ignore::airflow.exceptions.AirflowProviderDeprecationWarning")
+ def test_queue_and_trigger_queue_both_provided_raises(self):
+ """Test that providing both `queue` and deprecated `trigger_queue`
raises ValueError."""
+ with pytest.raises(ValueError, match="Cannot provide both `queue` and
deprecated `trigger_queue`"):
+ MessageQueueTrigger(scheme=PROVIDER_2_SCHEME, queue="q1",
trigger_queue="q2")
+
def test_scheme_provider_matching(self):
"""Test that scheme matching works correctly."""
provider1 = MockProvider(PROVIDER_1_NAME, PROVIDER_1_PATTERN,
scheme=PROVIDER_1_SCHEME)
@@ -331,14 +379,14 @@ class TestMessageQueueTriggerIntegration:
assert trigger_by_scheme is not None
def test_provider_integrations_validation(self):
- """Test that either queue or scheme parameter must be provided."""
- with pytest.raises(ValueError, match="Either `queue` or `scheme`
parameter must be provided"):
+ """Test that scheme parameter must be provided."""
+ with pytest.raises(ValueError, match="The `scheme` parameter must be
provided"):
MessageQueueTrigger()
def test_provider_integrations_deprecation_warning(self):
- """Test that a deprecation warning is raised when using the 'queue'
parameter."""
+ """Test that a deprecation warning is raised when using the 'queue'
parameter as broker URI."""
with pytest.warns(
AirflowProviderDeprecationWarning,
- match="The `queue` parameter is deprecated and will be removed in
future versions. Use the `scheme` parameter instead and pass configuration as
keyword arguments to `MessageQueueTrigger`.",
+ match="Passing the broker queue URI as `queue` is deprecated and
will be removed in future versions. Use the `scheme` parameter instead and pass
configuration as keyword arguments to `MessageQueueTrigger`.",
):
MessageQueueTrigger(queue="any queue")