This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 87fd884c0b dbt, openlineage: set run_id after defer, do not log error
if operator has no run_id set (#34270)
87fd884c0b is described below
commit 87fd884c0b9b90a02d454da323321f944a26c97e
Author: Maciej Obuchowski <[email protected]>
AuthorDate: Mon Sep 11 21:52:00 2023 +0200
dbt, openlineage: set run_id after defer, do not log error if operator has
no run_id set (#34270)
* dbt, openlineage: set run_id after defer, do not log error if operator
has no run_id set
Signed-off-by: Maciej Obuchowski <[email protected]>
* fix other tests
Signed-off-by: Maciej Obuchowski <[email protected]>
---------
Signed-off-by: Maciej Obuchowski <[email protected]>
---
airflow/providers/dbt/cloud/operators/dbt.py | 8 +++++---
.../providers/dbt/cloud/operators/test_dbt_cloud.py | 20 +++++++++++++++++---
tests/providers/dbt/cloud/utils/test_openlineage.py | 8 ++++++--
3 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/airflow/providers/dbt/cloud/operators/dbt.py
b/airflow/providers/dbt/cloud/operators/dbt.py
index a4c04a0919..5d8c6baca6 100644
--- a/airflow/providers/dbt/cloud/operators/dbt.py
+++ b/airflow/providers/dbt/cloud/operators/dbt.py
@@ -117,7 +117,7 @@ class DbtCloudRunJobOperator(BaseOperator):
self.timeout = timeout
self.check_interval = check_interval
self.additional_run_config = additional_run_config or {}
- self.run_id: int
+ self.run_id: int | None = None
self.deferrable = deferrable
def execute(self, context: Context):
@@ -135,12 +135,13 @@ class DbtCloudRunJobOperator(BaseOperator):
additional_run_config=self.additional_run_config,
)
self.run_id = trigger_job_response.json()["data"]["id"]
+ print(self.run_id)
job_run_url = trigger_job_response.json()["data"]["href"]
# Push the ``job_run_url`` value to XCom regardless of what happens
during execution so that the job
# run can be monitored via the operator link.
context["ti"].xcom_push(key="job_run_url", value=job_run_url)
- if self.wait_for_termination:
+ if self.wait_for_termination and isinstance(self.run_id, int):
if self.deferrable is False:
self.log.info("Waiting for job run %s to terminate.",
str(self.run_id))
@@ -197,6 +198,7 @@ class DbtCloudRunJobOperator(BaseOperator):
if event["status"] == "error":
raise AirflowException(event["message"])
self.log.info(event["message"])
+ self.run_id = event["run_id"]
return int(event["run_id"])
def on_kill(self) -> None:
@@ -225,7 +227,7 @@ class DbtCloudRunJobOperator(BaseOperator):
"""
from airflow.providers.openlineage.extractors import OperatorLineage
- if self.wait_for_termination is True:
+ if isinstance(self.run_id, int) and self.wait_for_termination is True:
return
generate_openlineage_events_from_dbt_cloud_run(operator=self,
task_instance=task_instance)
return OperatorLineage()
diff --git a/tests/providers/dbt/cloud/operators/test_dbt_cloud.py
b/tests/providers/dbt/cloud/operators/test_dbt_cloud.py
index 4400ebd4ab..17a929c66f 100644
--- a/tests/providers/dbt/cloud/operators/test_dbt_cloud.py
+++ b/tests/providers/dbt/cloud/operators/test_dbt_cloud.py
@@ -63,6 +63,12 @@ EXPLICIT_ACCOUNT_JOB_RUN_RESPONSE = {
}
+def mock_response_json(response: dict):
+ run_response = MagicMock(**response)
+ run_response.json.return_value = response
+ return run_response
+
+
def setup_module():
# Connection with ``account_id`` specified
conn_account_id = Connection(
@@ -125,7 +131,10 @@ class TestDbtCloudRunJobOperator:
)
@patch("airflow.providers.dbt.cloud.operators.dbt.DbtCloudRunJobOperator.defer")
@patch("airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.get_connection")
-
@patch("airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.trigger_job_run")
+ @patch(
+ "airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.trigger_job_run",
+ return_value=mock_response_json(DEFAULT_ACCOUNT_JOB_RUN_RESPONSE),
+ )
def test_execute_failed_before_getting_deferred(
self, mock_trigger_job_run, mock_dbt_hook, mock_defer,
mock_job_run_status
):
@@ -154,7 +163,10 @@ class TestDbtCloudRunJobOperator:
"airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.get_job_run_status",
)
@patch("airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.get_connection")
-
@patch("airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.trigger_job_run")
+ @patch(
+ "airflow.providers.dbt.cloud.hooks.dbt.DbtCloudHook.trigger_job_run",
+ return_value=mock_response_json(DEFAULT_ACCOUNT_JOB_RUN_RESPONSE),
+ )
def test_dbt_run_job_op_async(self, mock_trigger_job_run, mock_dbt_hook,
mock_job_run_status, status):
"""
Asserts that a task is deferred and an DbtCloudRunJobTrigger will be
fired
@@ -174,7 +186,9 @@ class TestDbtCloudRunJobOperator:
dbt_op.execute(MagicMock())
assert isinstance(exc.value.trigger, DbtCloudRunJobTrigger), "Trigger
is not a DbtCloudRunJobTrigger"
- @patch.object(DbtCloudHook, "trigger_job_run",
return_value=MagicMock(**DEFAULT_ACCOUNT_JOB_RUN_RESPONSE))
+ @patch.object(
+ DbtCloudHook, "trigger_job_run",
return_value=mock_response_json(DEFAULT_ACCOUNT_JOB_RUN_RESPONSE)
+ )
@pytest.mark.parametrize(
"job_run_status, expected_output",
[
diff --git a/tests/providers/dbt/cloud/utils/test_openlineage.py
b/tests/providers/dbt/cloud/utils/test_openlineage.py
index 0d93d5a2e1..ed9c7bfa40 100644
--- a/tests/providers/dbt/cloud/utils/test_openlineage.py
+++ b/tests/providers/dbt/cloud/utils/test_openlineage.py
@@ -22,6 +22,7 @@ from unittest.mock import MagicMock, patch
from airflow.providers.dbt.cloud.hooks.dbt import DbtCloudHook
from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator
from airflow.providers.dbt.cloud.utils.openlineage import
generate_openlineage_events_from_dbt_cloud_run
+from airflow.providers.openlineage.extractors import OperatorLineage
TASK_ID = "dbt_test"
DAG_ID = "dbt_dag"
@@ -130,7 +131,10 @@ class TestGenerateOpenLineageEventsFromDbtCloudRun:
)
mock_build_task_instance_run_id.return_value = TASK_UUID
-
generate_openlineage_events_from_dbt_cloud_run(mock_operator,
task_instance=mock_task_instance)
-
assert mock_client.emit.call_count == 4
+
+ def test_do_not_raise_error_if_runid_not_set_on_operator(self):
+ operator = DbtCloudRunJobOperator(task_id="dbt-job-runid-taskid",
job_id=1500)
+ assert operator.run_id is None
+ assert operator.get_openlineage_facets_on_complete(MagicMock()) ==
OperatorLineage()