YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager gets connection issue. Contributed by Xuan Gong
(cherry picked from commit 214fd1408c21f596d1d15217c11b58b34561aab7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c09bb465 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c09bb465 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c09bb465 Branch: refs/heads/branch-2.6 Commit: c09bb46579631a6e3b8dd50e964e77c9357ea14b Parents: cc30002 Author: Jian He <jia...@apache.org> Authored: Wed Sep 2 17:45:23 2015 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Fri Sep 25 16:26:18 2015 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/ha/ActiveStandbyElector.java | 4 ++++ hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/server/resourcemanager/AdminService.java | 9 +++++++++ .../yarn/server/resourcemanager/EmbeddedElectorService.java | 4 ++++ .../yarn/server/resourcemanager/webapp/AboutBlock.java | 2 ++ .../hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java | 5 +++++ .../yarn/server/resourcemanager/webapp/RMWebAppFilter.java | 5 ++++- .../yarn/server/resourcemanager/webapp/dao/ClusterInfo.java | 6 ++++++ .../server/resourcemanager/webapp/TestRMWebServices.java | 8 ++++++-- 9 files changed, 43 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java index 123f119..f8f720e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java @@ -1102,4 +1102,8 @@ public class ActiveStandbyElector implements StatCallback, StringCallback { ((appData == null) ? "null" : StringUtils.byteToHexString(appData)) + " cb=" + appClient; } + + public String getHAZookeeperConnectionState() { + return this.zkConnectionState.name(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a2e0653..6d016b7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -11,6 +11,9 @@ Release 2.6.2 - UNRELEASED YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. (Xuan Gong via jianhe) + YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager + gets connection issue. (Xuan Gong via jianhe) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 14109f9..1660e4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -722,4 +722,13 @@ public class AdminService extends CompositeService implements throw RPCUtil.getRemoteException(ioe); } } + + public String getHAZookeeperConnectionState() { + if (!rmContext.isHAEnabled()) { + return "ResourceManager HA is not enabled."; + } else if (!autoFailoverEnabled) { + return "Auto Failover is not enabled."; + } + return this.embeddedElector.getHAZookeeperConnectionState(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java index c7b7768..73bdca0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java @@ -205,4 +205,8 @@ public class EmbeddedElectorService extends AbstractService elector.quitElection(false); elector.joinElection(localActiveNodeInfo); } + + public String getHAZookeeperConnectionState() { + return elector.getHAZookeeperConnectionState(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java index ea5c48a..4225afd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java @@ -44,6 +44,8 @@ public class AboutBlock extends HtmlBlock { _("Cluster ID:", cinfo.getClusterId()). _("ResourceManager state:", cinfo.getState()). _("ResourceManager HA state:", cinfo.getHAState()). + _("ResourceManager HA zookeeper connection state:", + cinfo.getHAZookeeperConnectionState()). _("ResourceManager RMStateStore:", cinfo.getRMStateStore()). _("ResourceManager started on:", Times.format(cinfo.getStartedOn())). _("ResourceManager version:", cinfo.getRMBuildVersion() + http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 066ba52..2d19822 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -113,4 +113,9 @@ public class RMWebApp extends WebApp implements YarnWebParams { } return path; } + + public String getHAZookeeperConnectionState() { + return rm.getRMContext().getRMAdminService() + .getHAZookeeperConnectionState(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 61600dd..024a8d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -58,6 +58,7 @@ public class RMWebAppFilter extends GuiceContainer { private String path; private static final int BASIC_SLEEP_TIME = 5; private static final int MAX_SLEEP_TIME = 5 * 60; + private static final Random randnum = new Random(); @Inject public RMWebAppFilter(Injector injector, Configuration conf) { @@ -126,6 +127,8 @@ public class RMWebAppFilter extends GuiceContainer { String redirectMsg = doRetry ? "Can not find any active RM. Will retry in next " + next + " seconds." : "There is no active RM right now."; + redirectMsg += "\nHA Zookeeper Connection State: " + + rmWebApp.getHAZookeeperConnectionState(); PrintWriter out = response.getWriter(); out.println(redirectMsg); if (doRetry) { @@ -172,6 +175,6 @@ public class RMWebAppFilter extends GuiceContainer { private static int calculateExponentialTime(int retries) { long baseTime = BASIC_SLEEP_TIME * (1L << retries); - return (int) (baseTime * ((new Random()).nextDouble() + 0.5)); + return (int) (baseTime * (randnum.nextDouble() + 0.5)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java index b529f21..512a5c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java @@ -43,6 +43,7 @@ public class ClusterInfo { protected String hadoopVersion; protected String hadoopBuildVersion; protected String hadoopVersionBuiltOn; + protected String haZooKeeperConnectionState; public ClusterInfo() { } // JAXB needs this @@ -62,6 +63,8 @@ public class ClusterInfo { this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); + this.haZooKeeperConnectionState = + rm.getRMContext().getRMAdminService().getHAZookeeperConnectionState(); } public String getState() { @@ -108,4 +111,7 @@ public class ClusterInfo { return this.startedOn; } + public String getHAZookeeperConnectionState() { + return this.haZooKeeperConnectionState; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 9f091d2..bd31ff3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -261,6 +261,8 @@ public class TestRMWebServices extends JerseyTest { WebServicesTestUtils.getXmlLong(element, "startedOn"), WebServicesTestUtils.getXmlString(element, "state"), WebServicesTestUtils.getXmlString(element, "haState"), + WebServicesTestUtils.getXmlString( + element, "haZooKeeperConnectionState"), WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"), WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"), WebServicesTestUtils.getXmlString(element, "hadoopVersion"), @@ -276,9 +278,10 @@ public class TestRMWebServices extends JerseyTest { Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("clusterInfo"); - assertEquals("incorrect number of elements", 11, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"), info.getString("state"), info.getString("haState"), + info.getString("haZooKeeperConnectionState"), info.getString("hadoopVersionBuiltOn"), info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), info.getString("resourceManagerVersionBuiltOn"), @@ -288,7 +291,8 @@ public class TestRMWebServices extends JerseyTest { } public void verifyClusterGeneric(long clusterid, long startedon, - String state, String haState, String hadoopVersionBuiltOn, + String state, String haState, String haZooKeeperConnectionState, + String hadoopVersionBuiltOn, String hadoopBuildVersion, String hadoopVersion, String resourceManagerVersionBuiltOn, String resourceManagerBuildVersion, String resourceManagerVersion) {