Repository: ambari Updated Branches: refs/heads/trunk 7da4706a3 -> 8f58e5f97
AMBARI-15064. RU/EU can't start if hosts have name in MixedCASE in configs for NameNode, HBASE Master, ResourceManager (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8f58e5f9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8f58e5f9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8f58e5f9 Branch: refs/heads/trunk Commit: 8f58e5f97e0e6c7853956ad4f48d45603cee9d7f Parents: 7da4706 Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Thu Feb 18 11:21:04 2016 -0800 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Thu Feb 18 11:21:16 2016 -0800 ---------------------------------------------------------------------- .../controller/AmbariActionExecutionHelper.java | 13 +- .../internal/UpgradeResourceProvider.java | 1 + .../ambari/server/stack/MasterHostResolver.java | 53 +++++-- .../AmbariManagementControllerTest.java | 2 +- .../ambari/server/state/UpgradeHelperTest.java | 140 +++++++++++++++++-- 5 files changed, 178 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 795dfa7..88180c0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -262,6 +262,13 @@ public class AmbariActionExecutionHelper { final String serviceName = actionContext.getExpectedServiceName(); final String componentName = actionContext.getExpectedComponentName(); + LOG.debug("Called addExecutionCommandsToStage() for serviceName: {}, componentName: {}.", serviceName, componentName); + if (resourceFilter.getHostNames().isEmpty()) { + LOG.debug("Resource filter has no hostnames."); + } else { + LOG.debug("Resource filter has hosts: {}", StringUtils.join(resourceFilter.getHostNames(), ", ")); + } + if (null != cluster) { StackId stackId = cluster.getCurrentStackVersion(); if (serviceName != null && !serviceName.isEmpty()) { @@ -275,6 +282,7 @@ public class AmbariActionExecutionHelper { stackId.getStackVersion(), serviceName, componentName); } catch (ObjectNotFoundException e) { // do nothing, componentId is checked for null later + LOG.error("Did not find service {} and component {} in stack {}.", serviceName, componentName, stackId.getStackName()); } } else { for (String component : cluster.getService(serviceName).getServiceComponents().keySet()) { @@ -288,6 +296,7 @@ public class AmbariActionExecutionHelper { // All hosts are valid target host candidateHosts.addAll(clusters.getHostsForCluster(cluster.getClusterName()).keySet()); } + LOG.debug("Request for service {} and component {} is set to run on candidate hosts: {}.", serviceName, componentName, StringUtils.join(candidateHosts, ", ")); // Filter hosts that are in MS Set<String> ignoredHosts = maintenanceStateHelper.filterHostsInMaintenanceState( @@ -301,7 +310,9 @@ public class AmbariActionExecutionHelper { } } ); + if (! ignoredHosts.isEmpty()) { + LOG.debug("Hosts to ignore: {}.", StringUtils.join(ignoredHosts, ", ")); LOG.debug("Ignoring action for hosts due to maintenance state." + "Ignored hosts =" + ignoredHosts + ", component=" + componentName + ", service=" + serviceName @@ -323,7 +334,7 @@ public class AmbariActionExecutionHelper { for (String hostname : resourceFilter.getHostNames()) { if (!candidateHosts.contains(hostname)) { throw new AmbariException("Request specifies host " + hostname + - " but its not a valid host based on the " + + " but it is not a valid host based on the " + "target service=" + serviceName + " and component=" + componentName); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 70440fc..e5664c2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -1175,6 +1175,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider RequestResourceFilter filter = new RequestResourceFilter("", "", new ArrayList<String>(wrapper.getHosts())); + LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks()); Map<String, String> params = getNewParameterMap(); params.put(COMMAND_PARAM_TASKS, entity.getTasks()); params.put(COMMAND_PARAM_VERSION, context.getVersion()); http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java index 561350b..360f2b8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java @@ -63,7 +63,7 @@ public class MasterHostResolver { /** * Union of status for several services. */ - enum Status { + protected enum Status { ACTIVE, STANDBY } @@ -247,11 +247,11 @@ public class MasterHostResolver { return false; } - /** * Get mapping of the HDFS Namenodes from the state ("active" or "standby") to the hostname. * @return Returns a map from the state ("active" or "standby" to the hostname with that state if exactly * one active and one standby host were found, otherwise, return null. + * The hostnames are returned in lowercase. */ private Map<Status, String> getNameNodePair() { Map<Status, String> stateToHost = new HashMap<Status, String>(); @@ -291,7 +291,9 @@ public class MasterHostResolver { if (null != state && (state.equalsIgnoreCase(Status.ACTIVE.toString()) || state.equalsIgnoreCase(Status.STANDBY.toString()))) { Status status = Status.valueOf(state.toUpperCase()); - stateToHost.put(status, hp.host); + stateToHost.put(status, hp.host.toLowerCase()); + } else { + LOG.error(String.format("Could not retrieve state for NameNode %s from property %s by querying JMX.", hp.host, key)); } } catch (MalformedURLException e) { LOG.error(e.getMessage()); @@ -304,6 +306,12 @@ public class MasterHostResolver { return null; } + /** + * Resolve the name of the Resource Manager master and convert the hostname to lowercase. + * @param cluster Cluster + * @param hostType RM hosts + * @throws MalformedURLException + */ private void resolveResourceManagers(Cluster cluster, HostsType hostType) throws MalformedURLException { LinkedHashSet<String> orderedHosts = new LinkedHashSet<String>(hostType.hosts); @@ -320,18 +328,24 @@ public class MasterHostResolver { if (null != value) { if (null == hostType.master) { - hostType.master = hostname; + hostType.master = hostname.toLowerCase(); } // Quick and dirty to make sure the master is last in the list - orderedHosts.remove(hostname); - orderedHosts.add(hostname); + orderedHosts.remove(hostname.toLowerCase()); + orderedHosts.add(hostname.toLowerCase()); } } hostType.hosts = orderedHosts; } + /** + * Resolve the HBASE master and convert the hostname to lowercase. + * @param cluster Cluster + * @param hostsType HBASE master host. + * @throws AmbariException + */ private void resolveHBaseMasters(Cluster cluster, HostsType hostsType) throws AmbariException { String hbaseMasterInfoPortProperty = "hbase.master.info.port"; String hbaseMasterInfoPortValue = m_configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.HBASE_SITE, hbaseMasterInfoPortProperty); @@ -348,22 +362,31 @@ public class MasterHostResolver { if (null != value) { Boolean bool = Boolean.valueOf(value); if (bool.booleanValue()) { - hostsType.master = hostname; + hostsType.master = hostname.toLowerCase(); } else { - hostsType.secondary = hostname; + hostsType.secondary = hostname.toLowerCase(); } } - } } - private String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName, - boolean asQuery) { + protected String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName, + boolean asQuery) { return queryJmxBeanValue(hostname, port, beanName, attributeName, asQuery, false); } - private String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName, - boolean asQuery, boolean encrypted) { + /** + * Query the JMX attribute at http(s)://$server:$port/jmx?qry=$query or http(s)://$server:$port/jmx?get=$bean::$attribute + * @param hostname host name + * @param port port number + * @param beanName if asQuery is false, then search for this bean name + * @param attributeName if asQuery is false, then search for this attribute name + * @param asQuery whether to search bean or query + * @param encrypted true if using https instead of http. + * @return The jmx value. + */ + protected String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName, + boolean asQuery, boolean encrypted) { String protocol = encrypted ? "https://" : "http://"; String endPoint = protocol + (asQuery ? @@ -385,9 +408,9 @@ public class MasterHostResolver { return jmxBeans.get("beans").get(0).get(attributeName); } catch (Exception e) { if (LOG.isDebugEnabled()) { - LOG.info("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname, e); + LOG.debug("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname, e); } else { - LOG.info("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname); + LOG.debug("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index a0ac966..ffee3fa 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -4650,7 +4650,7 @@ public class AmbariManagementControllerTest { actionRequest = new ExecuteActionRequest("c1", null, "a2", resourceFilters, null, params, false); expectActionCreationErrorWithMessage(actionRequest, requestProperties, - "Request specifies host h6 but its not a valid host based on the target service=HDFS and component=DATANODE"); + "Request specifies host h6 but it is not a valid host based on the target service=HDFS and component=DATANODE"); hosts.clear(); hosts.add("h1"); http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index eb5bf62..4ea8f15 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -95,21 +95,23 @@ public class UpgradeHelperTest { private AmbariManagementController m_managementController; private Gson m_gson = new Gson(); - @Before - public void before() throws Exception { + /** + * Because test cases need to share config mocks, put common ones in this function. + * @throws Exception + */ + private void setConfigMocks() throws Exception { // configure the mock to return data given a specific placeholder m_configHelper = EasyMock.createNiceMock(ConfigHelper.class); + expect(m_configHelper.getPlaceholderValueFromDesiredConfigurations( + EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn("placeholder-rendered-properly").anyTimes(); + expect(m_configHelper.getEffectiveDesiredTags( + EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))).andReturn(new HashMap<String, Map<String, String>>()).anyTimes(); + } - expect( - m_configHelper.getPlaceholderValueFromDesiredConfigurations( - EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn( - "placeholder-rendered-properly").anyTimes(); - - expect( - m_configHelper.getEffectiveDesiredTags( - EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))). - andReturn(new HashMap<String, Map<String, String>>()).anyTimes(); - + @Before + public void before() throws Exception { + setConfigMocks(); + // Most test cases can replay the common config mocks. If any test case needs custom ones, it can re-initialize m_configHelper; replay(m_configHelper); final InMemoryDefaultTestModule injectorModule = new InMemoryDefaultTestModule() { @@ -1244,7 +1246,7 @@ public class UpgradeHelperTest { Service s = c.getService("ZOOKEEPER"); ServiceComponent sc = s.addServiceComponent("ZOOKEEPER_SERVER"); - ServiceComponentHost sch1 =sc.addServiceComponentHost("h1"); + ServiceComponentHost sch1 = sc.addServiceComponentHost("h1"); sch1.setVersion("2.1.1.0-1234"); ServiceComponentHost sch2 = sc.addServiceComponentHost("h2"); @@ -1252,7 +1254,6 @@ public class UpgradeHelperTest { List<ServiceComponentHost> schs = c.getServiceComponentHosts("ZOOKEEPER", "ZOOKEEPER_SERVER"); assertEquals(2, schs.size()); - MasterHostResolver mhr = new MasterHostResolver(null, c, "2.1.1.0-1234"); HostsType ht = mhr.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER"); @@ -1267,6 +1268,117 @@ public class UpgradeHelperTest { assertEquals("h2", ht.hosts.iterator().next()); } + /** + * Test that MasterHostResolver is case-insensitive even if configs have hosts in upper case for NameNode. + * @throws Exception + */ + @Test + public void testResolverCaseInsensitive() throws Exception { + Clusters clusters = injector.getInstance(Clusters.class); + ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); + + String clusterName = "c1"; + String version = "2.1.1.0-1234"; + + StackId stackId = new StackId("HDP-2.1.1"); + clusters.addCluster(clusterName, stackId); + Cluster c = clusters.getCluster(clusterName); + + helper.getOrCreateRepositoryVersion(stackId, + c.getDesiredStackVersion().getStackVersion()); + + c.createClusterVersion(stackId, + c.getDesiredStackVersion().getStackVersion(), "admin", + RepositoryVersionState.UPGRADING); + + for (int i = 0; i < 2; i++) { + String hostName = "h" + (i+1); + clusters.addHost(hostName); + Host host = clusters.getHost(hostName); + + Map<String, String> hostAttributes = new HashMap<String, String>(); + hostAttributes.put("os_family", "redhat"); + hostAttributes.put("os_release_version", "6"); + + host.setHostAttributes(hostAttributes); + + host.persist(); + clusters.mapHostToCluster(hostName, clusterName); + } + + // Add services + c.addService(serviceFactory.createNew(c, "HDFS")); + + Service s = c.getService("HDFS"); + ServiceComponent sc = s.addServiceComponent("NAMENODE"); + sc.addServiceComponentHost("h1"); + sc.addServiceComponentHost("h2"); + + List<ServiceComponentHost> schs = c.getServiceComponentHosts("HDFS", "NAMENODE"); + assertEquals(2, schs.size()); + + setConfigMocks(); + expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.nameservices")).andReturn("ha").anyTimes(); + expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes(); + expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes(); + + // Notice that these names are all caps. + expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes(); + expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes(); + replay(m_configHelper); + + MasterHostResolver mhr = new MockMasterHostResolver(m_configHelper, c, version); + + HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE"); + assertNotNull(ht.master); + assertNotNull(ht.secondary); + assertEquals(2, ht.hosts.size()); + + // Should be stored in lowercase. + assertTrue(ht.hosts.contains("h1")); + assertTrue(ht.hosts.contains("h1")); + } + + /** + * Extend {@link org.apache.ambari.server.stack.MasterHostResolver} in order to overwrite the JMX methods. + */ + private class MockMasterHostResolver extends MasterHostResolver { + + public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster) { + this(configHelper, cluster, null); + } + + public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version) { + super(configHelper, cluster, version); + } + + /** + * Mock the call to get JMX Values. + * @param hostname host name + * @param port port number + * @param beanName if asQuery is false, then search for this bean name + * @param attributeName if asQuery is false, then search for this attribute name + * @param asQuery whether to search bean or query + * @param encrypted true if using https instead of http. + * @return + */ + @Override + public String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName, + boolean asQuery, boolean encrypted) { + + if (beanName.equalsIgnoreCase("Hadoop:service=NameNode,name=NameNodeStatus") && attributeName.equalsIgnoreCase("State") && asQuery) { + switch (hostname) { + case "H1": + return Status.ACTIVE.toString(); + case "H2": + return Status.STANDBY.toString(); + default: + return "UNKNOWN_NAMENODE_STATUS_FOR_THIS_HOST"; + } + } + return "NOT_MOCKED"; + } + } private class MockModule implements Module {