Github user HyukjinKwon commented on the issue: https://github.com/apache/spark/pull/20404 How about something like this? ```diff --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -213,10 +213,10 @@ class SparkSession(object): self._jsc = self._sc._jsc self._jvm = self._sc._jvm if jsparkSession is None: - if self._jvm.SparkSession.getDefaultSession().isDefined() \ - and not self._jvm.SparkSession.getDefaultSession().get() \ - .sparkContext().isStopped(): - jsparkSession = self._jvm.SparkSession.getDefaultSession().get() + default_session = self._jvm.SparkSession.getDefaultSession() + if default_session.isDefined() \ + and not default_session.get().sparkContext().isStopped(): + jsparkSession = default_session else: jsparkSession = self._jvm.SparkSession(self._jsc.sc()) self._jsparkSession = jsparkSession @@ -230,7 +230,7 @@ class SparkSession(object): if SparkSession._instantiatedSession is None \ or SparkSession._instantiatedSession._sc._jsc is None: SparkSession._instantiatedSession = self - self._jvm.SparkSession.setDefaultSession(self._jsparkSession) + self._jvm.SparkSession.setDefaultSessionIfUnset(self._jsparkSession) def _repr_html_(self): return """ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index 734573ba31f..99747ef88bc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -1031,6 +1031,18 @@ object SparkSession { // Private methods from now on //////////////////////////////////////////////////////////////////////////////////////// + /** + * This if for Python blabla + */ + private[sql] def setDefaultSessionIfUnset(session: SparkSession): Unit = { + // Global synchronization blabla + SparkSession.synchronized { + if (defaultSession.get() eq null) { + defaultSession.set(session) + } + } + } + /** The active SparkSession for the current thread. */ private val activeThreadSession = new InheritableThreadLocal[SparkSession] ```
--- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org