This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9937361 [MINOR] Fix inconsistency log level among delegation token
providers
9937361 is described below
commit 993736154b6a46ffd7c3218173a2653a3842bba0
Author: Jungtaek Lim (HeartSaVioR) <kabh...@gmail.com>
AuthorDate: Tue Jan 1 09:14:23 2019 +0800
[MINOR] Fix inconsistency log level among delegation token providers
## What changes were proposed in this pull request?
There's some inconsistency for log level while logging error messages in
delegation token providers. (DEBUG, INFO, WARNING)
Given that failing to obtain token would often crash the query, I guess
it would be nice to set higher log level for error log messages.
## How was this patch tested?
The patch just changed the log level.
Closes #23418 from
HeartSaVioR/FIX-inconsistency-log-level-between-delegation-token-providers.
Authored-by: Jungtaek Lim (HeartSaVioR) <kabh...@gmail.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
.../security/HBaseDelegationTokenProvider.scala | 4 +-
.../security/HadoopFSDelegationTokenProvider.scala | 45 +++++++++++++---------
.../security/HiveDelegationTokenProvider.scala | 4 +-
.../security/KafkaDelegationTokenProvider.scala | 2 +-
4 files changed, 31 insertions(+), 24 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
index 5dcde4e..6ef6835 100644
---
a/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/security/HBaseDelegationTokenProvider.scala
@@ -50,7 +50,7 @@ private[security] class HBaseDelegationTokenProvider
creds.addToken(token.getService, token)
} catch {
case NonFatal(e) =>
- logDebug(s"Failed to get token from service $serviceName", e)
+ logWarning(s"Failed to get token from service $serviceName", e)
}
None
@@ -71,7 +71,7 @@ private[security] class HBaseDelegationTokenProvider
confCreate.invoke(null, conf).asInstanceOf[Configuration]
} catch {
case NonFatal(e) =>
- logDebug("Fail to invoke HBaseConfiguration", e)
+ logWarning("Fail to invoke HBaseConfiguration", e)
conf
}
}
diff --git
a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
index 767b552..00200f8 100644
---
a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.security
import scala.collection.JavaConverters._
import scala.util.Try
+import scala.util.control.NonFatal
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
@@ -44,28 +45,34 @@ private[deploy] class
HadoopFSDelegationTokenProvider(fileSystems: () => Set[Fil
hadoopConf: Configuration,
sparkConf: SparkConf,
creds: Credentials): Option[Long] = {
- val fsToGetTokens = fileSystems()
- val fetchCreds = fetchDelegationTokens(getTokenRenewer(hadoopConf),
fsToGetTokens, creds)
+ try {
+ val fsToGetTokens = fileSystems()
+ val fetchCreds = fetchDelegationTokens(getTokenRenewer(hadoopConf),
fsToGetTokens, creds)
- // Get the token renewal interval if it is not set. It will only be called
once.
- if (tokenRenewalInterval == null) {
- tokenRenewalInterval = getTokenRenewalInterval(hadoopConf, sparkConf,
fsToGetTokens)
- }
+ // Get the token renewal interval if it is not set. It will only be
called once.
+ if (tokenRenewalInterval == null) {
+ tokenRenewalInterval = getTokenRenewalInterval(hadoopConf, sparkConf,
fsToGetTokens)
+ }
- // Get the time of next renewal.
- val nextRenewalDate = tokenRenewalInterval.flatMap { interval =>
- val nextRenewalDates = fetchCreds.getAllTokens.asScala
-
.filter(_.decodeIdentifier().isInstanceOf[AbstractDelegationTokenIdentifier])
- .map { token =>
- val identifier = token
- .decodeIdentifier()
- .asInstanceOf[AbstractDelegationTokenIdentifier]
- identifier.getIssueDate + interval
- }
- if (nextRenewalDates.isEmpty) None else Some(nextRenewalDates.min)
- }
+ // Get the time of next renewal.
+ val nextRenewalDate = tokenRenewalInterval.flatMap { interval =>
+ val nextRenewalDates = fetchCreds.getAllTokens.asScala
+
.filter(_.decodeIdentifier().isInstanceOf[AbstractDelegationTokenIdentifier])
+ .map { token =>
+ val identifier = token
+ .decodeIdentifier()
+ .asInstanceOf[AbstractDelegationTokenIdentifier]
+ identifier.getIssueDate + interval
+ }
+ if (nextRenewalDates.isEmpty) None else Some(nextRenewalDates.min)
+ }
- nextRenewalDate
+ nextRenewalDate
+ } catch {
+ case NonFatal(e) =>
+ logWarning(s"Failed to get token from service $serviceName", e)
+ None
+ }
}
override def delegationTokensRequired(
diff --git
a/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala
index 7249eb8..90f7051 100644
---
a/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/security/HiveDelegationTokenProvider.scala
@@ -49,7 +49,7 @@ private[spark] class HiveDelegationTokenProvider
new HiveConf(hadoopConf, classOf[HiveConf])
} catch {
case NonFatal(e) =>
- logDebug("Fail to create Hive Configuration", e)
+ logWarning("Fail to create Hive Configuration", e)
hadoopConf
case e: NoClassDefFoundError =>
logWarning(classNotFoundErrorStr)
@@ -104,7 +104,7 @@ private[spark] class HiveDelegationTokenProvider
None
} catch {
case NonFatal(e) =>
- logDebug(s"Failed to get token from service $serviceName", e)
+ logWarning(s"Failed to get token from service $serviceName", e)
None
case e: NoClassDefFoundError =>
logWarning(classNotFoundErrorStr)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/security/KafkaDelegationTokenProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/security/KafkaDelegationTokenProvider.scala
index 45995be..f67cb26 100644
---
a/core/src/main/scala/org/apache/spark/deploy/security/KafkaDelegationTokenProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/security/KafkaDelegationTokenProvider.scala
@@ -44,7 +44,7 @@ private[security] class KafkaDelegationTokenProvider
return Some(nextRenewalDate)
} catch {
case NonFatal(e) =>
- logInfo(s"Failed to get token from service $serviceName", e)
+ logWarning(s"Failed to get token from service $serviceName", e)
}
None
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
