This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8bf2515 [SPARK-37729][SQL] Fix SparkSession.setLogLevel that is not
working in Spark Shell
8bf2515 is described below
commit 8bf2515a768d5216f36737b23537285e29247a25
Author: Liang-Chi Hsieh <vii...@gmail.com>
AuthorDate: Fri Dec 24 17:33:25 2021 +0900
[SPARK-37729][SQL] Fix SparkSession.setLogLevel that is not working in
Spark Shell
### What changes were proposed in this pull request?
This patch fixes a regression by upgrading to log4j 2.
`SparkSession.setLogLevel` should work in spark-shell.
### Why are the changes needed?
Currently in spark-shell `SparkSession.setLogLevel` can not work well.
Previously log4j 1.x API only calls the API on logger to set logging level.
Now with log4j 2.x, the updated logging level at root logger is not taken by
other loggers. We need to explicitly ask updating loggers with the updated
logger config.
### Does this PR introduce _any_ user-facing change?
No, the upgrading is not released yet.
### How was this patch tested?
Manual test and unit test.
Closes #35012 from viirya/SPARK-37729.
Authored-by: Liang-Chi Hsieh <vii...@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
core/src/main/scala/org/apache/spark/util/Utils.scala | 14 +++++++++-----
core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 3 +++
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala
b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 996af86b..4c7160c 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -57,6 +57,8 @@ import
org.apache.hadoop.io.compress.{CompressionCodecFactory, SplittableCompres
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.util.{RunJar, StringUtils}
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.logging.log4j.{Level, LogManager}
+import org.apache.logging.log4j.core.LoggerContext
import org.eclipse.jetty.util.MultiException
import org.slf4j.Logger
@@ -2423,11 +2425,13 @@ private[spark] object Utils extends Logging {
/**
* configure a new log4j level
*/
- def setLogLevel(l: org.apache.logging.log4j.Level): Unit = {
- val rootLogger = org.apache.logging.log4j.LogManager.getRootLogger()
- .asInstanceOf[org.apache.logging.log4j.core.Logger]
- rootLogger.setLevel(l)
- rootLogger.get().setLevel(l)
+ def setLogLevel(l: Level): Unit = {
+ val ctx = LogManager.getContext(false).asInstanceOf[LoggerContext]
+ val config = ctx.getConfiguration()
+ val loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME)
+ loggerConfig.setLevel(l)
+ ctx.updateLoggers()
+
// Setting threshold to null as rootLevel will define log level for
spark-shell
Logging.sparkShellThresholdLevel = null
}
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 3c5f500..bfb1bc3 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -690,8 +690,11 @@ class UtilsSuite extends SparkFunSuite with
ResetSystemProperties with Logging {
try {
Utils.setLogLevel(org.apache.logging.log4j.Level.ALL)
assert(rootLogger.getLevel == org.apache.logging.log4j.Level.ALL)
+ assert(log.isInfoEnabled())
Utils.setLogLevel(org.apache.logging.log4j.Level.ERROR)
assert(rootLogger.getLevel == org.apache.logging.log4j.Level.ERROR)
+ assert(!log.isInfoEnabled())
+ assert(log.isErrorEnabled())
} finally {
// Best effort at undoing changes this test made.
Utils.setLogLevel(current)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
