SameerMesiah97 commented on code in PR #71661:
URL: https://github.com/apache/airflow/pull/71661#discussion_r3792259733
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -1385,10 +1385,28 @@ def on_kill(self) -> None:
# state because `yarn_track_via_rm_api=True` deliberately terminates
# `_submit_sp` right after submission to free the JVM.
if self._yarn_application_id and self._yarn_track_via_rm_api:
- self._kill_yarn_application(self._yarn_application_id)
+ self.kill_yarn_application(self._yarn_application_id)
self._run_post_submit_commands()
+ @property
+ def conf(self) -> dict[str, Any]:
+ """Return the live Spark conf dict (not a copy); callers may mutate
keys in place."""
+ return self._conf
+
+ @property
+ def kubernetes_driver_pod(self) -> str | None:
+ return self._kubernetes_driver_pod
+
+ @kubernetes_driver_pod.setter
+ def kubernetes_driver_pod(self, value: str | None) -> None:
+ self._kubernetes_driver_pod = value
+
+ @property
+ def yarn_application_id(self) -> str | None:
+ """YARN application id captured from spark-submit logs. Get-only."""
+ return self._yarn_application_id
Review Comment:
I am not sure about exposing these through properties while keeping the
underlying attributes private. Since `conf` exposes the mutable dict directly
and `kubernetes_driver_pod` has both a getter and setter, what encapsulation do
we gain by retaining the private attributes?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
