moomindani commented on code in PR #68519:
URL: https://github.com/apache/airflow/pull/68519#discussion_r3421242232
##########
providers/databricks/src/airflow/providers/databricks/operators/databricks.py:
##########
@@ -992,26 +1121,32 @@ def _get_hook(self, caller: str) -> DatabricksHook:
)
def execute(self, context: Context):
+ json = self._get_merged_json()
+ self._validate_merged_json(json)
hook = self._hook
- if "job_name" in self.json:
- job_id = hook.find_job_id_by_name(self.json["job_name"])
+ if "job_name" in json:
+ job_id = hook.find_job_id_by_name(json["job_name"])
if job_id is None:
- raise AirflowException(f"Job ID for job name
{self.json['job_name']} can not be found")
- self.json["job_id"] = job_id
- del self.json["job_name"]
+ raise DatabricksOperatorPayloadError(
+ f"Job ID for job name {json['job_name']} can not be found"
+ )
+ json["job_id"] = job_id
+ del json["job_name"]
if self.cancel_previous_runs:
- if (job_id := self.json.get("job_id")) is None:
+ if (job_id := json.get("job_id")) is None:
raise ValueError(
"cancel_previous_runs=True requires either job_id or
job_name to be provided."
)
hook.cancel_all_runs(job_id)
- if not self.json.get("job_parameters") and self.params:
- self.json["job_parameters"] = dict(self.params)
+ json = cast("dict[str, Any]", normalise_json_content(json))
Review Comment:
`normalise_json_content` — which raises on invalid payload types — is called
here, after `find_job_id_by_name` and potentially `cancel_all_runs` have
already made Databricks API calls. In contrast, `DatabricksCreateJobsOperator`
calls it before any API interaction.
This inconsistency means a malformed payload is only detected after remote
state may have been modified (a job looked up, runs cancelled). Move this call
to before `hook = self._hook` so all local validation completes first.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
