This is an automated email from the ASF dual-hosted git repository.
pankajkoti pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c377e7fceb add warning log when task_key>100 (#42813)
c377e7fceb is described below
commit c377e7fcebcbf2b2752e937d9b797864329333ef
Author: Kalyan R <[email protected]>
AuthorDate: Tue Oct 8 21:40:18 2024 +0530
add warning log when task_key>100 (#42813)
related to #41816
Adds a warning log to indicate failure if length of task_key>100.
---
airflow/providers/databricks/operators/databricks.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/databricks/operators/databricks.py
b/airflow/providers/databricks/operators/databricks.py
index d9256477f9..58ffaeaece 100644
--- a/airflow/providers/databricks/operators/databricks.py
+++ b/airflow/providers/databricks/operators/databricks.py
@@ -1045,7 +1045,15 @@ class DatabricksTaskBaseOperator(BaseOperator, ABC):
def _get_databricks_task_id(self, task_id: str) -> str:
"""Get the databricks task ID using dag_id and task_id. Removes
illegal characters."""
- return f"{self.dag_id}__{task_id.replace('.', '__')}"
+ task_id = f"{self.dag_id}__{task_id.replace('.', '__')}"
+ if len(task_id) > 100:
+ self.log.warning(
+ "The generated task_key '%s' exceeds 100 characters and will
be truncated by the Databricks API. "
+ "This will cause failure when trying to monitor the task.
task_key is generated by ",
+ "concatenating dag_id and task_id.",
+ task_id,
+ )
+ return task_id
@property
def _databricks_workflow_task_group(self) -> DatabricksWorkflowTaskGroup |
None:
