This is an automated email from the ASF dual-hosted git repository.
dabla pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ff9a6a926ce Add durable execution to
AzureDataFactoryRunPipelineOperator (#77162)
ff9a6a926ce is described below
commit ff9a6a926ce3bcc4866ff69a26b34a520a4692ca
Author: Amogh Desai <[email protected]>
AuthorDate: Tue Nov 17 12:27:19 2026 +0530
Add durable execution to AzureDataFactoryRunPipelineOperator (#77162)
* Add durable execution to AzureDataFactoryRunPipelineOperator
---
providers/microsoft/azure/docs/changelog.rst | 8 ++
.../azure/docs/operators/adf_run_pipeline.rst | 28 ++++
.../microsoft/azure/operators/data_factory.py | 78 +++++++++--
.../microsoft/azure/triggers/data_factory.py | 7 +-
.../microsoft/azure/operators/test_data_factory.py | 149 ++++++++++++++++++++-
.../unit/microsoft/azure/operators/test_powerbi.py | 15 +--
.../azure/tests/unit/microsoft/azure/test_utils.py | 13 ++
.../microsoft/azure/triggers/test_data_factory.py | 16 ++-
8 files changed, 282 insertions(+), 32 deletions(-)
diff --git a/providers/microsoft/azure/docs/changelog.rst
b/providers/microsoft/azure/docs/changelog.rst
index 5a7bbe0dad3..aa563aca807 100644
--- a/providers/microsoft/azure/docs/changelog.rst
+++ b/providers/microsoft/azure/docs/changelog.rst
@@ -27,6 +27,14 @@
Changelog
---------
+.. note::
+
+ On Airflow 3.3 and later, ``AzureDataFactoryRunPipelineOperator`` is
durable by default: a retry
+ reconnects to the stored pipeline run instead of starting a new one, and
returns without running the
+ pipeline again when that run has succeeded. On Airflow 3.3 and 3.4,
clearing the whole Dag run keeps the
+ stored run id, so a task whose run succeeded does not re-run the pipeline.
Clear the task instance
+ instead, or set ``durable=False`` to start a new run on every try. Below
Airflow 3.3 nothing changes.
+
15.3.1
......
diff --git a/providers/microsoft/azure/docs/operators/adf_run_pipeline.rst
b/providers/microsoft/azure/docs/operators/adf_run_pipeline.rst
index ee4a468bafa..96403b4411e 100644
--- a/providers/microsoft/azure/docs/operators/adf_run_pipeline.rst
+++ b/providers/microsoft/azure/docs/operators/adf_run_pipeline.rst
@@ -62,6 +62,34 @@ Also you can use deferrable mode in
:class:`~airflow.providers.microsoft.azure.s
:start-after: [START howto_operator_adf_run_pipeline_async]
:end-before: [END howto_operator_adf_run_pipeline_async]
+Durable execution
+~~~~~~~~~~~~~~~~~
+
+The operator is :doc:`durable <apache-airflow:core-concepts/resumable-tasks>`
by default: it keeps the
+pipeline run id in the :doc:`task state store
<apache-airflow:core-concepts/task-state-store>` before it
+waits. A retry reconnects to a run that is still in progress, in both regular
and deferrable mode, and does
+not start a new one once the run has succeeded. A run that failed, was
cancelled, is being cancelled or no
+longer exists is started again. With ``wait_for_termination=False``, a retry
returns without waiting when the
+stored run is still in progress. In deferrable mode, an error while polling no
longer cancels the run, so
+the retry can reconnect to it. Durable execution requires Airflow 3.3 or newer.
+
+Clearing the task instance from the UI or the REST API discards the stored run
id by default, so the next
+try starts a new pipeline run. To keep the stored run id instead, tick "keep
task state" in the clear
+dialog or pass ``keep_task_state=true`` to the clear endpoint. The next try
then treats the stored run
+like a retry does.
+
+Before Airflow 3.5, clearing the whole Dag run, marking a task as success or
failed, and
+``airflow tasks clear`` / ``airflow dags clear`` keep the stored run id. So if
you clear the whole Dag run,
+a task whose pipeline run succeeded returns without running the pipeline
again. Clear the task instance
+itself instead. From Airflow 3.5, every way of clearing discards the stored
run id by default.
+
+Clearing a task that is waiting on the worker cancels its pipeline run first,
so the next try starts a new
+run either way. Only a deferred task's run keeps going. See
:ref:`apache-airflow:concepts-resumable-tasks`
+for details.
+
+Set ``durable=False`` to start a new pipeline run on every try, and to cancel
the run when polling fails in
+deferrable mode. Below Airflow 3.3 the operator behaves as with
``durable=False``.
+
Poll for status of a data factory pipeline run asynchronously
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py
index 36321dbe10a..ad1030fbee9 100644
---
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py
+++
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py
@@ -22,11 +22,14 @@ from collections.abc import Sequence
from functools import cached_property
from typing import TYPE_CHECKING, Any
+from azure.core.exceptions import ResourceNotFoundError
+
from airflow.providers.common.compat.sdk import (
AirflowException,
BaseHook,
BaseOperator,
BaseOperatorLink,
+ ResumableJobMixin,
XCom,
conf,
)
@@ -40,6 +43,8 @@ from airflow.providers.microsoft.azure.triggers.data_factory
import AzureDataFac
from airflow.utils.log.logging_mixin import LoggingMixin
if TYPE_CHECKING:
+ from pydantic import JsonValue
+
from airflow.providers.common.compat.sdk import TaskInstanceKey
from airflow.sdk import Context
@@ -80,7 +85,7 @@ class AzureDataFactoryPipelineRunLink(LoggingMixin,
BaseOperatorLink):
return url
-class AzureDataFactoryRunPipelineOperator(BaseOperator):
+class AzureDataFactoryRunPipelineOperator(ResumableJobMixin, BaseOperator):
"""
Execute a data factory pipeline.
@@ -115,8 +120,13 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
:param check_interval: Time in seconds to check on a pipeline run's status
for non-asynchronous waits.
Used only if ``wait_for_termination`` is True.
:param deferrable: Run operator in deferrable mode.
+ :param durable: Keep the pipeline run id in the task state store, so that
a retry reconnects to the
+ run still in progress, or returns when it has succeeded, instead of
starting a new one (default
+ is True). Needs Airflow 3.3 or later; below it the operator behaves as
with ``durable=False``.
"""
+ external_id_key = "azure_data_factory_pipeline_run_id"
+
template_fields: Sequence[str] = (
"azure_data_factory_conn_id",
"resource_group_name",
@@ -147,8 +157,12 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
timeout: int = 60 * 60 * 24 * 7,
check_interval: int = 60,
deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
+ durable: bool | None = None,
**kwargs,
) -> None:
+ # Named param so default_args reaches it below Airflow 3.3 too.
+ if durable is not None:
+ kwargs["durable"] = durable
super().__init__(**kwargs)
self.azure_data_factory_conn_id = azure_data_factory_conn_id
self.pipeline_name = pipeline_name
@@ -163,6 +177,8 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
self.timeout = timeout
self.check_interval = check_interval
self.deferrable = deferrable
+ self.run_id: str | None = None
+ self._pushed_run_id: str | None = None
@cached_property
def hook(self) -> AzureDataFactoryHook:
@@ -170,6 +186,9 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
return
AzureDataFactoryHook(azure_data_factory_conn_id=self.azure_data_factory_conn_id)
def execute(self, context: Context) -> None:
+ self.execute_resumable(context)
+
+ def submit_job(self, context: Context) -> str:
self.log.info("Executing the %s pipeline.", self.pipeline_name)
response = self.hook.run_pipeline(
self.pipeline_name,
@@ -182,52 +201,72 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
parameters=self.parameters,
)
self.run_id = response.run_id
- # Push the ``run_id`` value to XCom regardless of what happens during
execution. This allows for
- # retrieval the executed pipeline's ``run_id`` for downstream tasks
especially if performing an
- # asynchronous wait.
- context["ti"].xcom_push(key="run_id", value=self.run_id)
+ return response.run_id
+
+ def get_job_status(self, external_id: JsonValue, context: Context) -> str:
+ try:
+ return self.hook.get_pipeline_run_status(
+ str(external_id), self.resource_group_name, self.factory_name
+ )
+ except ResourceNotFoundError:
+ return "NotFound"
+
+ def is_job_active(self, status: str) -> bool:
+ # A Canceling run can only end Cancelled, so the retry starts a new
run instead of waiting on it.
+ return status in (
+ AzureDataFactoryPipelineRunStatus.QUEUED,
+ AzureDataFactoryPipelineRunStatus.IN_PROGRESS,
+ )
+
+ def is_job_succeeded(self, status: str) -> bool:
+ return status == AzureDataFactoryPipelineRunStatus.SUCCEEDED
+
+ def poll_until_complete(self, external_id: JsonValue, context: Context) ->
None:
+ run_id = str(external_id)
+ self._record_run_id(run_id, context)
if self.wait_for_termination:
if self.deferrable is False:
- self.log.info("Waiting for pipeline run %s to terminate.",
self.run_id)
+ self.log.info("Waiting for pipeline run %s to terminate.",
run_id)
if self.hook.wait_for_pipeline_run_status(
- self.run_id,
+ run_id,
AzureDataFactoryPipelineRunStatus.SUCCEEDED,
self.resource_group_name,
self.factory_name,
check_interval=self.check_interval,
timeout=self.timeout,
):
- self.log.info("Pipeline run %s has completed
successfully.", self.run_id)
+ self.log.info("Pipeline run %s has completed
successfully.", run_id)
else:
raise AzureDataFactoryPipelineRunException(
- f"Pipeline run {self.run_id} has failed or has been
cancelled."
+ f"Pipeline run {run_id} has failed or has been
cancelled."
)
else:
end_time = time.time() + self.timeout
pipeline_run_status = self.hook.get_pipeline_run_status(
- self.run_id, self.resource_group_name, self.factory_name
+ run_id, self.resource_group_name, self.factory_name
)
if pipeline_run_status not in
AzureDataFactoryPipelineRunStatus.TERMINAL_STATUSES:
self.defer(
timeout=self.execution_timeout,
trigger=AzureDataFactoryTrigger(
azure_data_factory_conn_id=self.azure_data_factory_conn_id,
- run_id=self.run_id,
+ run_id=run_id,
wait_for_termination=self.wait_for_termination,
resource_group_name=self.resource_group_name,
factory_name=self.factory_name,
check_interval=self.check_interval,
end_time=end_time,
+ cancel_on_error=not self.durable,
),
method_name="execute_complete",
)
elif pipeline_run_status ==
AzureDataFactoryPipelineRunStatus.SUCCEEDED:
- self.log.info("Pipeline run %s has completed
successfully.", self.run_id)
+ self.log.info("Pipeline run %s has completed
successfully.", run_id)
elif pipeline_run_status in
AzureDataFactoryPipelineRunStatus.FAILURE_STATES:
raise AzureDataFactoryPipelineRunException(
- f"Pipeline run {self.run_id} has failed or has been
cancelled."
+ f"Pipeline run {run_id} has failed or has been
cancelled."
)
else:
if self.deferrable is True:
@@ -238,6 +277,19 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
stacklevel=2,
)
+ def get_job_result(self, external_id: JsonValue, context: Context) -> None:
+ self._record_run_id(str(external_id), context)
+
+ def _record_run_id(self, run_id: str, context: Context) -> None:
+ self.run_id = run_id
+ if self._pushed_run_id == run_id:
+ return
+ self._pushed_run_id = run_id
+ # Push the ``run_id`` value to XCom regardless of what happens during
execution. This allows for
+ # retrieval the executed pipeline's ``run_id`` for downstream tasks
especially if performing an
+ # asynchronous wait.
+ context["ti"].xcom_push(key="run_id", value=run_id)
+
def execute_complete(self, context: Context, event: dict[str, str]) ->
None:
"""
Return immediately - callback for when the trigger fires.
diff --git
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/triggers/data_factory.py
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/triggers/data_factory.py
index 7e0d3e4b334..6c5fa428a52 100644
---
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/triggers/data_factory.py
+++
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/triggers/data_factory.py
@@ -130,6 +130,8 @@ class AzureDataFactoryTrigger(BaseTrigger):
:param factory_name: The data factory name.
:param wait_for_termination: Flag to wait on a pipeline run's termination.
:param check_interval: Time in seconds to check on a pipeline run's status.
+ :param cancel_on_error: Cancel the pipeline run when polling fails
unexpectedly (default is True).
+ A durable operator passes False, so that its retry reconnects to the
run instead.
"""
def __init__(
@@ -141,6 +143,7 @@ class AzureDataFactoryTrigger(BaseTrigger):
factory_name: str,
wait_for_termination: bool = True,
check_interval: int = 60,
+ cancel_on_error: bool = True,
):
super().__init__()
self.azure_data_factory_conn_id = azure_data_factory_conn_id
@@ -150,6 +153,7 @@ class AzureDataFactoryTrigger(BaseTrigger):
self.resource_group_name = resource_group_name
self.factory_name = factory_name
self.end_time = end_time
+ self.cancel_on_error = cancel_on_error
def serialize(self) -> tuple[str, dict[str, Any]]:
"""Serialize AzureDataFactoryTrigger arguments and classpath."""
@@ -163,6 +167,7 @@ class AzureDataFactoryTrigger(BaseTrigger):
"resource_group_name": self.resource_group_name,
"factory_name": self.factory_name,
"end_time": self.end_time,
+ "cancel_on_error": self.cancel_on_error,
},
)
@@ -245,7 +250,7 @@ class AzureDataFactoryTrigger(BaseTrigger):
)
except Exception as e:
self.log.exception(e)
- if self.run_id:
+ if self.run_id and self.cancel_on_error:
try:
self.log.info("Cancelling pipeline run %s",
self.run_id)
await hook.cancel_pipeline_run(
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_data_factory.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_data_factory.py
index ac31c8b40b8..d87a21e45be 100644
---
a/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_data_factory.py
+++
b/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_data_factory.py
@@ -24,6 +24,8 @@ from unittest.mock import MagicMock, patch
import pendulum
import pytest
+from azure.core.exceptions import ResourceNotFoundError
+from azure.mgmt.datafactory.models import CreateRunResponse
from airflow.models import DAG, Connection
from airflow.models.dagrun import DagRun
@@ -39,7 +41,8 @@ from airflow.providers.microsoft.azure.triggers.data_factory
import AzureDataFac
from airflow.utils.types import DagRunType
from tests_common.test_utils.taskinstance import create_task_instance as
_create_task_instance
-from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
+from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS,
AIRFLOW_V_3_3_PLUS
+from unit.microsoft.azure.test_utils import TaskStateStore
if AIRFLOW_V_3_0_PLUS:
from airflow.sdk.execution_time.comms import XComResult
@@ -66,6 +69,9 @@ EXPECTED_PIPELINE_RUN_OP_EXTRA_LINK = (
"factories/{factory_name}"
)
AZ_PIPELINE_RUN_ID = "7f8c6c72-c093-11ec-a83d-0242ac120007"
+STORED_RUN_ID = "stored-run-id"
+NEW_RUN_ID = "new-run-id"
+RUN_ID_KEY = "azure_data_factory_pipeline_run_id"
class TestAzureDataFactoryRunPipelineOperator:
@@ -496,3 +502,144 @@ class
TestAzureDataFactoryRunPipelineOperatorWithDeferrable:
context={},
event={"status": "error", "message": "error", "run_id":
AZ_PIPELINE_RUN_ID},
)
+
+
[email protected](not AIRFLOW_V_3_3_PLUS, reason="The task state store
arrived in Airflow 3.3")
+class TestAzureDataFactoryRunPipelineOperatorDurable:
+ CONFIG = {
+ "task_id": TASK_ID,
+ "azure_data_factory_conn_id": AZURE_DATA_FACTORY_CONN_ID,
+ "pipeline_name": PIPELINE_NAME,
+ "resource_group_name": "resource-group-name",
+ "factory_name": "factory-name",
+ "check_interval": 1,
+ "timeout": 3,
+ }
+
+ @pytest.fixture(autouse=True)
+ def setup_mocks(self):
+ with (
+ patch.object(
+ AzureDataFactoryHook,
+ "run_pipeline",
+ autospec=True,
+ return_value=MagicMock(spec=CreateRunResponse,
run_id=NEW_RUN_ID),
+ ) as self.mock_run_pipeline,
+ patch.object(AzureDataFactoryHook, "get_pipeline_run_status",
autospec=True) as self.mock_status,
+ patch.object(
+ AzureDataFactoryHook, "wait_for_pipeline_run_status",
autospec=True, return_value=True
+ ) as self.mock_wait,
+ ):
+ yield
+
+ @staticmethod
+ def make_context(**stored):
+ return {"ti": MagicMock(spec=TaskInstance), "task_state_store":
TaskStateStore(**stored)}
+
+ def test_execute_stores_run_id_of_fresh_submit(self):
+ context = self.make_context()
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG)
+
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_called_once()
+ assert context["task_state_store"].values == {RUN_ID_KEY: NEW_RUN_ID}
+ context["ti"].xcom_push.assert_called_once_with(key="run_id",
value=NEW_RUN_ID)
+
+ @pytest.mark.parametrize(
+ "status", [AzureDataFactoryPipelineRunStatus.QUEUED,
AzureDataFactoryPipelineRunStatus.IN_PROGRESS]
+ )
+ def test_execute_reconnects_to_run_in_progress(self, status):
+ self.mock_status.return_value = status
+ context = self.make_context(**{RUN_ID_KEY: STORED_RUN_ID})
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG)
+
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_not_called()
+ assert self.mock_wait.call_args.args[1] == STORED_RUN_ID
+ assert operator.run_id == STORED_RUN_ID
+ context["ti"].xcom_push.assert_called_once_with(key="run_id",
value=STORED_RUN_ID)
+
+ def test_execute_deferrable_reconnects_to_run_in_progress(self):
+ self.mock_status.return_value =
AzureDataFactoryPipelineRunStatus.IN_PROGRESS
+ context = self.make_context(**{RUN_ID_KEY: STORED_RUN_ID})
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG,
deferrable=True)
+
+ with pytest.raises(TaskDeferred) as exc:
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_not_called()
+ assert isinstance(exc.value.trigger, AzureDataFactoryTrigger)
+ assert exc.value.trigger.run_id == STORED_RUN_ID
+ assert exc.value.trigger.cancel_on_error is False
+ context["ti"].xcom_push.assert_called_once_with(key="run_id",
value=STORED_RUN_ID)
+
+ def test_execute_deferrable_stores_run_id_before_deferring(self):
+ self.mock_status.return_value =
AzureDataFactoryPipelineRunStatus.IN_PROGRESS
+ context = self.make_context()
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG,
deferrable=True)
+
+ with pytest.raises(TaskDeferred):
+ operator.execute(context)
+
+ assert context["task_state_store"].values == {RUN_ID_KEY: NEW_RUN_ID}
+
+ @pytest.mark.parametrize("deferrable", [False, True])
+ def test_execute_skips_succeeded_run(self, deferrable):
+ self.mock_status.return_value =
AzureDataFactoryPipelineRunStatus.SUCCEEDED
+ context = self.make_context(**{RUN_ID_KEY: STORED_RUN_ID})
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG,
deferrable=deferrable)
+
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_not_called()
+ self.mock_wait.assert_not_called()
+ assert operator.run_id == STORED_RUN_ID
+ context["ti"].xcom_push.assert_called_once_with(key="run_id",
value=STORED_RUN_ID)
+
+ @pytest.mark.parametrize(
+ "stored_status",
+ [
+ AzureDataFactoryPipelineRunStatus.FAILED,
+ AzureDataFactoryPipelineRunStatus.CANCELLED,
+ AzureDataFactoryPipelineRunStatus.CANCELING,
+ "NotFound",
+ ],
+ )
+ def test_execute_resubmits_when_stored_run_cannot_be_resumed(self,
stored_status):
+ if stored_status == "NotFound":
+ self.mock_status.side_effect = ResourceNotFoundError("not found")
+ else:
+ self.mock_status.return_value = stored_status
+ context = self.make_context(**{RUN_ID_KEY: STORED_RUN_ID})
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG)
+
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_called_once()
+ assert context["task_state_store"].values == {RUN_ID_KEY: NEW_RUN_ID}
+ assert operator.run_id == NEW_RUN_ID
+
+ def test_execute_without_waiting_returns_for_run_in_progress(self):
+ self.mock_status.return_value =
AzureDataFactoryPipelineRunStatus.IN_PROGRESS
+ context = self.make_context(**{RUN_ID_KEY: STORED_RUN_ID})
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG,
wait_for_termination=False)
+
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_not_called()
+ self.mock_wait.assert_not_called()
+ context["ti"].xcom_push.assert_called_once_with(key="run_id",
value=STORED_RUN_ID)
+
+ def test_execute_not_durable_always_submits(self):
+ self.mock_status.return_value =
AzureDataFactoryPipelineRunStatus.IN_PROGRESS
+ context = self.make_context(**{RUN_ID_KEY: STORED_RUN_ID})
+ operator = AzureDataFactoryRunPipelineOperator(**self.CONFIG,
durable=False)
+
+ operator.execute(context)
+
+ self.mock_run_pipeline.assert_called_once()
+ self.mock_status.assert_not_called()
+ assert context["task_state_store"].values == {RUN_ID_KEY:
STORED_RUN_ID}
+ context["ti"].xcom_push.assert_called_once_with(key="run_id",
value=NEW_RUN_ID)
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_powerbi.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_powerbi.py
index 40879b33332..88fd5081e85 100644
---
a/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_powerbi.py
+++
b/providers/microsoft/azure/tests/unit/microsoft/azure/operators/test_powerbi.py
@@ -36,7 +36,7 @@ from airflow.providers.microsoft.azure.triggers.powerbi
import PowerBITrigger
from tests_common.test_utils.operators.run_deferrable import run_trigger
from tests_common.test_utils.version_compat import AIRFLOW_V_3_3_PLUS
-from unit.microsoft.azure.test_utils import get_airflow_connection
+from unit.microsoft.azure.test_utils import TaskStateStore,
get_airflow_connection
DEFAULT_CONNECTION_CLIENT_SECRET = "powerbi_conn_id"
TASK_ID = "run_powerbi_operator"
@@ -105,19 +105,6 @@ def previous_refresh(status: str) -> dict[str, str]:
}
-class TaskStateStore:
- """In-memory stand-in for ``context["task_state_store"]``."""
-
- def __init__(self, **values):
- self.values = dict(values)
-
- def get(self, key, default=None):
- return self.values.get(key, default)
-
- def set(self, key, value, *, retention=None):
- self.values[key] = value
-
-
class TestPowerBIDatasetRefreshOperator:
@mock.patch.object(BaseHook, "get_connection",
side_effect=get_airflow_connection)
def test_handle_refresh_wait_for_completion(self, connection):
diff --git a/providers/microsoft/azure/tests/unit/microsoft/azure/test_utils.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/test_utils.py
index 7b8cf56e912..612138858a9 100644
--- a/providers/microsoft/azure/tests/unit/microsoft/azure/test_utils.py
+++ b/providers/microsoft/azure/tests/unit/microsoft/azure/test_utils.py
@@ -303,3 +303,16 @@ def mock_authentication_provider(closed: bool | None =
None) -> AuthenticationPr
credential=mock_token_credentials(closed=closed)
)
return
BaseBearerTokenAuthenticationProvider(access_token_provider=access_token_provider)
+
+
+class TaskStateStore:
+ """In-memory stand-in for ``context["task_state_store"]``."""
+
+ def __init__(self, **values):
+ self.values = dict(values)
+
+ def get(self, key, default=None):
+ return self.values.get(key, default)
+
+ def set(self, key, value, *, retention=None):
+ self.values[key] = value
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_data_factory.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_data_factory.py
index c43d9e3a5e0..8beaf884138 100644
---
a/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_data_factory.py
+++
b/providers/microsoft/azure/tests/unit/microsoft/azure/triggers/test_data_factory.py
@@ -248,6 +248,7 @@ class TestAzureDataFactoryTrigger:
"end_time": AZ_PIPELINE_END_TIME,
"wait_for_termination": True,
"check_interval": 60,
+ "cancel_on_error": True,
}
@pytest.mark.asyncio
@@ -369,15 +370,24 @@ class TestAzureDataFactoryTrigger:
assert expected == actual
@pytest.mark.asyncio
+ @pytest.mark.parametrize(("cancel_on_error", "cancel_calls"), [(True, 1),
(False, 0)])
@mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.cancel_pipeline_run")
@mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_adf_pipeline_run_status")
async def test_azure_data_factory_trigger_run_exception(
- self, mock_pipeline_run_status, mock_cancel_pipeline_run
+ self, mock_pipeline_run_status, mock_cancel_pipeline_run,
cancel_on_error, cancel_calls
):
"""Assert that run catch exception if Azure API throw exception"""
mock_pipeline_run_status.side_effect = Exception("Test exception")
+ trigger = AzureDataFactoryTrigger(
+ run_id=AZ_PIPELINE_RUN_ID,
+ resource_group_name=AZ_RESOURCE_GROUP_NAME,
+ factory_name=AZ_FACTORY_NAME,
+ azure_data_factory_conn_id=AZ_DATA_FACTORY_CONN_ID,
+ end_time=AZ_PIPELINE_END_TIME,
+ cancel_on_error=cancel_on_error,
+ )
- task = [i async for i in self.TRIGGER.run()]
+ task = [i async for i in trigger.run()]
response = TriggerEvent(
{
"status": "error",
@@ -387,7 +397,7 @@ class TestAzureDataFactoryTrigger:
)
assert len(task) == 1
assert response in task
- mock_cancel_pipeline_run.assert_called_once()
+ assert mock_cancel_pipeline_run.call_count == cancel_calls
@pytest.mark.asyncio
@mock.patch(f"{MODULE}.hooks.data_factory.AzureDataFactoryAsyncHook.get_adf_pipeline_run_status")