HyukjinKwon commented on code in PR #45232: URL: https://github.com/apache/spark/pull/45232#discussion_r1510818206
########## python/pyspark/resource/profile.py: ########## @@ -114,14 +122,26 @@ def id(self) -> int: int A unique id of this :class:`ResourceProfile` """ - - if self._java_resource_profile is not None: - return self._java_resource_profile.id() - else: - raise RuntimeError( - "SparkContext must be created to get the id, get the id " - "after adding the ResourceProfile to an RDD" - ) + with self._lock: + if self._id is None: + if self._java_resource_profile is not None: + self._id = self._java_resource_profile.id() + else: + from pyspark.sql import is_remote + + if is_remote(): + from pyspark.sql.connect.resource.profile import _ResourceProfile + + rp = _ResourceProfile( + self._executor_resource_requests, self._task_resource_requests + ) + self._id = rp.id + else: + raise RuntimeError( + "SparkContext must be created to get the id, get the id " + "after adding the ResourceProfile to an RDD" Review Comment: RDD is hidden in Spark Connect concept. Let's probably avoid this term. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org