This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 57c921193b Fix failed tasks are not detected in `AzureBatchHook`
(#36785)
57c921193b is described below
commit 57c921193b86569f42bb521a356d28d1c4c619f3
Author: ArvidMartensRenson
<[email protected]>
AuthorDate: Tue Jan 16 10:41:05 2024 +0100
Fix failed tasks are not detected in `AzureBatchHook` (#36785)
* fix wait_for_job_tasks_to_complete in microsoft-azure provider
* add unit tests for wait_for_all_tasks_to_complete
* fix azure batch hook unit tests
* fix azure batch hook tests
---
airflow/providers/microsoft/azure/hooks/batch.py | 4 +-
.../providers/microsoft/azure/hooks/test_batch.py | 58 ++++++++++++++++++++--
2 files changed, 57 insertions(+), 5 deletions(-)
diff --git a/airflow/providers/microsoft/azure/hooks/batch.py
b/airflow/providers/microsoft/azure/hooks/batch.py
index 7b681b3a9d..2d26cc9f03 100644
--- a/airflow/providers/microsoft/azure/hooks/batch.py
+++ b/airflow/providers/microsoft/azure/hooks/batch.py
@@ -378,7 +378,7 @@ class AzureBatchHook(BaseHook):
"""
timeout_time = timezone.utcnow() + timedelta(minutes=timeout)
while timezone.utcnow() < timeout_time:
- tasks = self.connection.task.list(job_id)
+ tasks = list(self.connection.task.list(job_id))
incomplete_tasks = [task for task in tasks if task.state !=
batch_models.TaskState.completed]
if not incomplete_tasks:
@@ -386,7 +386,7 @@ class AzureBatchHook(BaseHook):
fail_tasks = [
task
for task in tasks
- if task.executionInfo.result ==
batch_models.TaskExecutionResult.failure
+ if task.execution_info.result ==
batch_models.TaskExecutionResult.failure
]
return fail_tasks
for task in incomplete_tasks:
diff --git a/tests/providers/microsoft/azure/hooks/test_batch.py
b/tests/providers/microsoft/azure/hooks/test_batch.py
index d9daa2f13d..dd696911b2 100644
--- a/tests/providers/microsoft/azure/hooks/test_batch.py
+++ b/tests/providers/microsoft/azure/hooks/test_batch.py
@@ -190,9 +190,61 @@ class TestAzureBatchHook:
mock_instance.assert_called_once_with(job_id="myjob", task=task)
@mock.patch(f"{MODULE}.BatchServiceClient")
- def test_wait_for_all_task_to_complete(self, mock_batch):
- # TODO: Add test
- pass
+ def test_wait_for_all_task_to_complete_timeout(self, mock_batch):
+ hook = AzureBatchHook(azure_batch_conn_id=self.test_cloud_conn_id)
+ with pytest.raises(TimeoutError):
+ hook.wait_for_job_tasks_to_complete("myjob", -1)
+
+ @mock.patch(f"{MODULE}.BatchServiceClient")
+ def test_wait_for_all_task_to_complete_all_success(self, mock_batch):
+ hook = AzureBatchHook(azure_batch_conn_id=self.test_cloud_conn_id)
+ hook.connection.task.list.return_value = iter(
+ [
+ batch_models.CloudTask(
+ id="mytask_1",
+ execution_info=batch_models.TaskExecutionInformation(
+ retry_count=0, requeue_count=0,
result=batch_models.TaskExecutionResult.success
+ ),
+ state=batch_models.TaskState.completed,
+ ),
+ batch_models.CloudTask(
+ id="mytask_2",
+ execution_info=batch_models.TaskExecutionInformation(
+ retry_count=0, requeue_count=0,
result=batch_models.TaskExecutionResult.success
+ ),
+ state=batch_models.TaskState.completed,
+ ),
+ ]
+ )
+
+ results = hook.wait_for_job_tasks_to_complete("myjob", 60)
+ assert results == []
+ hook.connection.task.list.assert_called_once_with("myjob")
+
+ @mock.patch(f"{MODULE}.BatchServiceClient")
+ def test_wait_for_all_task_to_complete_failures(self, mock_batch):
+ hook = AzureBatchHook(azure_batch_conn_id=self.test_cloud_conn_id)
+ tasks = [
+ batch_models.CloudTask(
+ id="mytask_1",
+ execution_info=batch_models.TaskExecutionInformation(
+ retry_count=0, requeue_count=0,
result=batch_models.TaskExecutionResult.success
+ ),
+ state=batch_models.TaskState.completed,
+ ),
+ batch_models.CloudTask(
+ id="mytask_2",
+ execution_info=batch_models.TaskExecutionInformation(
+ retry_count=0, requeue_count=0,
result=batch_models.TaskExecutionResult.failure
+ ),
+ state=batch_models.TaskState.completed,
+ ),
+ ]
+ hook.connection.task.list.return_value = iter(tasks)
+
+ results = hook.wait_for_job_tasks_to_complete("myjob", 60)
+ assert results == [tasks[1]]
+ hook.connection.task.list.assert_called_once_with("myjob")
@mock.patch(f"{MODULE}.BatchServiceClient")
def test_connection_success(self, mock_batch):