Repository: spark Updated Branches: refs/heads/master 157fe64f3 -> 5fcf4c2bf
[SPARK-12316] Wait a minutes to avoid cycle calling. When application end, AM will clean the staging dir. But if the driver trigger to update the delegation token, it will can't find the right token file and then it will endless cycle call the method 'updateCredentialsIfRequired'. Then it lead driver StackOverflowError. https://issues.apache.org/jira/browse/SPARK-12316 Author: huangzhaowei <carlmartin...@gmail.com> Closes #10475 from SaintBacchus/SPARK-12316. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fcf4c2b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fcf4c2b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fcf4c2b Branch: refs/heads/master Commit: 5fcf4c2bfce4b7e3543815c8e49ffdec8072c9a2 Parents: 157fe64 Author: huangzhaowei <carlmartin...@gmail.com> Authored: Thu Feb 25 09:14:19 2016 -0600 Committer: Tom Graves <tgra...@yahoo-inc.com> Committed: Thu Feb 25 09:14:19 2016 -0600 ---------------------------------------------------------------------- .../spark/deploy/yarn/ExecutorDelegationTokenUpdater.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/5fcf4c2b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorDelegationTokenUpdater.scala ---------------------------------------------------------------------- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorDelegationTokenUpdater.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorDelegationTokenUpdater.scala index 9d99c0d..6474acc 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorDelegationTokenUpdater.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorDelegationTokenUpdater.scala @@ -76,7 +76,10 @@ private[spark] class ExecutorDelegationTokenUpdater( SparkHadoopUtil.get.getTimeFromNowToRenewal( sparkConf, 0.8, UserGroupInformation.getCurrentUser.getCredentials) if (timeFromNowToRenewal <= 0) { - executorUpdaterRunnable.run() + // We just checked for new credentials but none were there, wait a minute and retry. + // This handles the shutdown case where the staging directory may have been removed(see + // SPARK-12316 for more details). + delegationTokenRenewer.schedule(executorUpdaterRunnable, 1, TimeUnit.MINUTES) } else { logInfo(s"Scheduling token refresh from HDFS in $timeFromNowToRenewal millis.") delegationTokenRenewer.schedule( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org