YARN-4087. Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs. Contributed by Jian He
(cherry picked from commit 9f7fcb54e798cf4fda1ea7972dd96491976e1857) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1828ba00 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1828ba00 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1828ba00 Branch: refs/heads/branch-2.6 Commit: 1828ba00be40721ae798e2288a75a5e7d187b643 Parents: d27f09c Author: Xuan <xg...@apache.org> Authored: Mon Sep 7 18:09:35 2015 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Fri Sep 25 16:43:06 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../src/main/resources/yarn-default.xml | 5 ++++- .../resourcemanager/recovery/RMStateStore.java | 15 +++++++++------ 4 files changed, 17 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 15d082b..fdf65bb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -21,6 +21,9 @@ Release 2.6.2 - UNRELEASED BUG FIXES + YARN-4087. Followup fixes after YARN-2019 regarding RM behavior when + state-store error occurs. (Jian He via xgong) + Release 2.6.1 - 2015-09-23 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 471297c..d7f24ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -381,7 +381,7 @@ public class YarnConfiguration extends Configuration { public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast"; - public static final boolean DEFAULT_YARN_FAIL_FAST = true; + public static final boolean DEFAULT_YARN_FAIL_FAST = false; public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index fa2e5cc..9b56d99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -286,9 +286,12 @@ <property> <description>Should YARN fail fast if it encounters any errors. + This is a global config for all other components including RM,NM etc. + If no value is set for component-specific config (e.g yarn.resourcemanager.fail-fast), + this value will be the default. </description> <name>yarn.fail-fast</name> - <value>true</value> + <value>false</value> </property> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 37855f7..fa912ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -822,17 +823,19 @@ public abstract class RMStateStore extends AbstractService { */ protected void notifyStoreOperationFailed(Exception failureCause) { LOG.error("State store operation failed ", failureCause); - if (failureCause instanceof StoreFencedException) { + if (HAUtil.isHAEnabled(getConfig())) { + LOG.warn("State-store fenced ! Transitioning RM to standby"); Thread standByTransitionThread = new Thread(new StandByTransitionThread()); standByTransitionThread.setName("StandByTransitionThread Handler"); standByTransitionThread.start(); + } else if (YarnConfiguration.shouldRMFailFast(getConfig())) { + LOG.fatal("Fail RM now due to state-store error!"); + rmDispatcher.getEventHandler().handle( + new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED, + failureCause)); } else { - if (YarnConfiguration.shouldRMFailFast(getConfig())) { - rmDispatcher.getEventHandler().handle( - new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED, - failureCause)); - } + LOG.warn("Skip the state-store error."); } }
