spark git commit: [SPARK-25076][SQL] SQLConf should not be retrieved from a stopped SparkSession
Repository: spark
Updated Branches:
refs/heads/branch-2.3 7d465d8f4 -> 9bfc55b1b
[SPARK-25076][SQL] SQLConf should not be retrieved from a stopped SparkSession
## What changes were proposed in this pull request?
When a `SparkSession` is stopped, `SQLConf.get` should use the fallback conf to
avoid weird issues like
```
sbt.ForkMain$ForkError: java.lang.IllegalStateException: LiveListenerBus is
stopped.
at
org.apache.spark.scheduler.LiveListenerBus.addToQueue(LiveListenerBus.scala:97)
at
org.apache.spark.scheduler.LiveListenerBus.addToStatusQueue(LiveListenerBus.scala:80)
at
org.apache.spark.sql.internal.SharedState.(SharedState.scala:93)
at
org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:120)
at
org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:120)
at scala.Option.getOrElse(Option.scala:121)
...
```
## How was this patch tested?
a new test suite
Closes #22056 from cloud-fan/session.
Authored-by: Wenchen Fan
Signed-off-by: Xiao Li
(cherry picked from commit fec67ed7e95483c5ea97a7b263ad4bea7d3d42b5)
Signed-off-by: Xiao Li
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9bfc55b1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9bfc55b1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9bfc55b1
Branch: refs/heads/branch-2.3
Commit: 9bfc55b1b0aae269320bb978027a800fd1878149
Parents: 7d465d8
Author: Wenchen Fan
Authored: Thu Aug 9 14:38:58 2018 -0700
Committer: Xiao Li
Committed: Thu Aug 9 14:40:09 2018 -0700
--
.../org/apache/spark/sql/SparkSession.scala | 3 +-
.../apache/spark/sql/LocalSparkSession.scala| 9 ++
.../spark/sql/internal/SQLConfGetterSuite.scala | 33
3 files changed, 37 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/9bfc55b1/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index b699ccd..adc7143 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -92,7 +92,8 @@ class SparkSession private(
// If there is no active SparkSession, uses the default SQL conf. Otherwise,
use the session's.
SQLConf.setSQLConfGetter(() => {
-
SparkSession.getActiveSession.map(_.sessionState.conf).getOrElse(SQLConf.getFallbackConf)
+
SparkSession.getActiveSession.filterNot(_.sparkContext.isStopped).map(_.sessionState.conf)
+ .getOrElse(SQLConf.getFallbackConf)
})
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/9bfc55b1/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
b/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
index cbef1c7..6b90f20 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
@@ -36,19 +36,14 @@ trait LocalSparkSession extends BeforeAndAfterEach with
BeforeAndAfterAll { self
override def afterEach() {
try {
- resetSparkContext()
+ LocalSparkSession.stop(spark)
SparkSession.clearActiveSession()
SparkSession.clearDefaultSession()
+ spark = null
} finally {
super.afterEach()
}
}
-
- def resetSparkContext(): Unit = {
-LocalSparkSession.stop(spark)
-spark = null
- }
-
}
object LocalSparkSession {
http://git-wip-us.apache.org/repos/asf/spark/blob/9bfc55b1/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
new file mode 100644
index 000..bb79d3a
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *
spark git commit: [SPARK-25076][SQL] SQLConf should not be retrieved from a stopped SparkSession
Repository: spark
Updated Branches:
refs/heads/master bd6db1505 -> fec67ed7e
[SPARK-25076][SQL] SQLConf should not be retrieved from a stopped SparkSession
## What changes were proposed in this pull request?
When a `SparkSession` is stopped, `SQLConf.get` should use the fallback conf to
avoid weird issues like
```
sbt.ForkMain$ForkError: java.lang.IllegalStateException: LiveListenerBus is
stopped.
at
org.apache.spark.scheduler.LiveListenerBus.addToQueue(LiveListenerBus.scala:97)
at
org.apache.spark.scheduler.LiveListenerBus.addToStatusQueue(LiveListenerBus.scala:80)
at
org.apache.spark.sql.internal.SharedState.(SharedState.scala:93)
at
org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:120)
at
org.apache.spark.sql.SparkSession$$anonfun$sharedState$1.apply(SparkSession.scala:120)
at scala.Option.getOrElse(Option.scala:121)
...
```
## How was this patch tested?
a new test suite
Closes #22056 from cloud-fan/session.
Authored-by: Wenchen Fan
Signed-off-by: Xiao Li
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fec67ed7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fec67ed7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fec67ed7
Branch: refs/heads/master
Commit: fec67ed7e95483c5ea97a7b263ad4bea7d3d42b5
Parents: bd6db15
Author: Wenchen Fan
Authored: Thu Aug 9 14:38:58 2018 -0700
Committer: Xiao Li
Committed: Thu Aug 9 14:38:58 2018 -0700
--
.../org/apache/spark/sql/SparkSession.scala | 3 +-
.../apache/spark/sql/LocalSparkSession.scala| 9 ++
.../spark/sql/internal/SQLConfGetterSuite.scala | 33
3 files changed, 37 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fec67ed7/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
--
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 565042f..d9278d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -92,7 +92,8 @@ class SparkSession private(
// If there is no active SparkSession, uses the default SQL conf. Otherwise,
use the session's.
SQLConf.setSQLConfGetter(() => {
-
SparkSession.getActiveSession.map(_.sessionState.conf).getOrElse(SQLConf.getFallbackConf)
+
SparkSession.getActiveSession.filterNot(_.sparkContext.isStopped).map(_.sessionState.conf)
+ .getOrElse(SQLConf.getFallbackConf)
})
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/fec67ed7/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
b/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
index cbef1c7..6b90f20 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/LocalSparkSession.scala
@@ -36,19 +36,14 @@ trait LocalSparkSession extends BeforeAndAfterEach with
BeforeAndAfterAll { self
override def afterEach() {
try {
- resetSparkContext()
+ LocalSparkSession.stop(spark)
SparkSession.clearActiveSession()
SparkSession.clearDefaultSession()
+ spark = null
} finally {
super.afterEach()
}
}
-
- def resetSparkContext(): Unit = {
-LocalSparkSession.stop(spark)
-spark = null
- }
-
}
object LocalSparkSession {
http://git-wip-us.apache.org/repos/asf/spark/blob/fec67ed7/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
--
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
new file mode 100644
index 000..bb79d3a
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfGetterSuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to
