Vamsi-klu commented on code in PR #71661:
URL: https://github.com/apache/airflow/pull/71661#discussion_r3792598896
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -1385,10 +1385,28 @@ def on_kill(self) -> None:
# state because `yarn_track_via_rm_api=True` deliberately terminates
# `_submit_sp` right after submission to free the JVM.
if self._yarn_application_id and self._yarn_track_via_rm_api:
- self._kill_yarn_application(self._yarn_application_id)
+ self.kill_yarn_application(self._yarn_application_id)
self._run_post_submit_commands()
+ @property
+ def conf(self) -> dict[str, Any]:
+ """Return the live Spark conf dict (not a copy); callers may mutate
keys in place."""
+ return self._conf
+
+ @property
+ def kubernetes_driver_pod(self) -> str | None:
+ return self._kubernetes_driver_pod
+
+ @kubernetes_driver_pod.setter
+ def kubernetes_driver_pod(self, value: str | None) -> None:
+ self._kubernetes_driver_pod = value
+
+ @property
+ def yarn_application_id(self) -> str | None:
+ """YARN application id captured from spark-submit logs. Get-only."""
+ return self._yarn_application_id
Review Comment:
Good catch. Those properties were not really hiding anything, so I dropped
them. conf and kubernetes_driver_pod are regular public attributes now.
yarn_application_id is still get-only, since only the hook should set that from
the spark-submit logs. I also did not keep the old _conf /
_kubernetes_driver_pod names around. Those were never a public API.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
