Repository: ambari Updated Branches: refs/heads/branch-2.4 297183521 -> 40b1560e0
AMBARI-16072. Stack Advisor issue when adding service to Kerberized cluster (rlevas) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/40b1560e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/40b1560e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/40b1560e Branch: refs/heads/branch-2.4 Commit: 40b1560e050115908c2248e5d539a1cc46ce9c74 Parents: 2971835 Author: Robert Levas <[email protected]> Authored: Fri May 13 14:45:51 2016 -0400 Committer: Robert Levas <[email protected]> Committed: Fri May 13 14:45:51 2016 -0400 ---------------------------------------------------------------------- .../server/controller/KerberosHelperImpl.java | 3 +- .../org/apache/ambari/server/state/Cluster.java | 13 ++ .../server/state/cluster/ClusterImpl.java | 37 +++ .../server/controller/KerberosHelperTest.java | 21 +- .../server/state/cluster/ClusterTest.java | 223 +++++++++++++++++++ 5 files changed, 289 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/40b1560e/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java index fe466d5..15bd2d9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java @@ -432,7 +432,7 @@ public class KerberosHelperImpl implements KerberosHelper { setAuthToLocalRules(kerberosDescriptor, cluster, kerberosDetails.getDefaultRealm(), configurations, kerberosConfigurations); return (applyStackAdvisorUpdates) - ? applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore, + ? applyStackAdvisorUpdates(cluster, cluster.getServices().keySet(), configurations, kerberosConfigurations, propertiesToIgnore, null, null, kerberosEnabled) : kerberosConfigurations; @@ -513,6 +513,7 @@ public class KerberosHelperImpl implements KerberosHelper { .forStack(stackVersion.getStackName(), stackVersion.getStackVersion()) .forServices(new ArrayList<String>(services)) .forHosts(hostNames) + .withComponentHostsMap(cluster.getServiceComponentHostMap(null, services)) .withConfigurations(requestConfigurations) .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS) .build(); http://git-wip-us.apache.org/repos/asf/ambari/blob/40b1560e/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index 9c0052f..d9df8b4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -89,6 +89,19 @@ public interface Cluster { List<ServiceComponentHost> getServiceComponentHosts(String hostname); /** + * Gets a map of components to hosts they are installed on. + * <p> + * This may may be filtered by host and/or service by optionally providing a set of hostname + * and/or service names to use as a filter. <code>null</code> for either filter indicates no + * filter (or all), an empty set indicates a complete filter (or none). + * + * @param hostNames + * @param serviceNames + * @return a map of (filtered) components to hosts + */ + Map<String, Set<String>> getServiceComponentHostMap(Set<String> hostNames, Set<String> serviceNames); + + /** * Get all ServiceComponentHosts for a given service and optional component * * If the component name is <code>null</code>, all components for the requested service will be returned. http://git-wip-us.apache.org/repos/asf/ambari/blob/40b1560e/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 13eea58..6cf9933 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -920,6 +920,43 @@ public class ClusterImpl implements Cluster { } @Override + public Map<String, Set<String>> getServiceComponentHostMap(Set<String> hostNames, Set<String> serviceNames) { + Map<String, Set<String>> componentHostMap = new HashMap<String, Set<String>>(); + + Collection<Host> hosts = getHosts(); + + if(hosts != null) { + for (Host host : hosts) { + String hostname = host.getHostName(); + + // If this host is not filtered out, continue processing + if ((hostNames == null) || hostNames.contains(hostname)) { + List<ServiceComponentHost> serviceComponentHosts = getServiceComponentHosts(hostname); + + if (serviceComponentHosts != null) { + for (ServiceComponentHost sch : serviceComponentHosts) { + // If the service for this ServiceComponentHost is not filtered out, continue processing + if ((serviceNames == null) || serviceNames.contains(sch.getServiceName())) { + String component = sch.getServiceComponentName(); + Set<String> componentHosts = componentHostMap.get(component); + + if (componentHosts == null) { + componentHosts = new HashSet<String>(); + componentHostMap.put(component, componentHosts); + } + + componentHosts.add(hostname); + } + } + } + } + } + } + + return componentHostMap; + } + + @Override public List<ServiceComponentHost> getServiceComponentHosts(String serviceName, String componentName) { ArrayList<ServiceComponentHost> foundItems = new ArrayList<ServiceComponentHost>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/40b1560e/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java index e03ed86..5393fd6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java @@ -2343,6 +2343,18 @@ public class KerberosHelperTest extends EasyMockSupport { } }); + Map<String, Service> services = new HashMap<String, Service>(); + services.put("SERVICE1", service1); + services.put("SERVICE2", service2); + services.put("SERVICE3", service3); + + Map<String, Set<String>> serviceComponentHostMap = new HashMap<String, Set<String>>(); + serviceComponentHostMap.put("COMPONENT1A", Collections.singleton("hostA")); + serviceComponentHostMap.put("COMPONENT1B", new HashSet<String>(Arrays.asList("hostB", "hostC"))); + serviceComponentHostMap.put("COMPONENT2A", Collections.singleton("hostA")); + serviceComponentHostMap.put("COMPONENT2B", new HashSet<String>(Arrays.asList("hostB", "hostC"))); + serviceComponentHostMap.put("COMPONEN3A", Collections.singleton("hostA")); + final Cluster cluster = createMock(Cluster.class); expect(cluster.getDesiredConfigByType("krb5-conf")).andReturn(krb5ConfConfig).atLeastOnce(); expect(cluster.getDesiredConfigByType("kerberos-env")).andReturn(kerberosEnvConfig).atLeastOnce(); @@ -2350,13 +2362,8 @@ public class KerberosHelperTest extends EasyMockSupport { expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.2")).atLeastOnce(); expect(cluster.getClusterName()).andReturn("c1").atLeastOnce(); expect(cluster.getHosts()).andReturn(hosts).anyTimes(); - expect(cluster.getServices()).andReturn(new HashMap<String, Service>() { - { - put("SERVICE1", service1); - put("SERVICE2", service2); - put("SERVICE3", service3); - } - }).anyTimes(); + expect(cluster.getServices()).andReturn(services).anyTimes(); + expect(cluster.getServiceComponentHostMap(null, services.keySet())).andReturn(serviceComponentHostMap).anyTimes(); expect(cluster.isBluePrintDeployed()).andReturn(false).atLeastOnce(); final Map<String, Map<String, String>> existingConfigurations = new HashMap<String, Map<String, String>>() { http://git-wip-us.apache.org/repos/asf/ambari/blob/40b1560e/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java index 7dc8bf8..826ea65 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java @@ -826,6 +826,229 @@ public class ClusterTest { } @Test + public void testGetServiceComponentHostMap() throws Exception { + createDefaultCluster(); + + Service s = serviceFactory.createNew(c1, "HDFS"); + c1.addService(s); + s.persist(); + + ServiceComponent scNN = serviceComponentFactory.createNew(s, "NAMENODE"); + s.addServiceComponent(scNN); + scNN.persist(); + ServiceComponentHost schNNH1 = serviceComponentHostFactory.createNew(scNN, "h1"); + scNN.addServiceComponentHost(schNNH1); + schNNH1.persist(); + + ServiceComponent scDN = serviceComponentFactory.createNew(s, "DATANODE"); + s.addServiceComponent(scDN); + scDN.persist(); + ServiceComponentHost scDNH1 = serviceComponentHostFactory.createNew(scDN, "h1"); + scDN.addServiceComponentHost(scDNH1); + scDNH1.persist(); + ServiceComponentHost scDNH2 = serviceComponentHostFactory.createNew(scDN, "h2"); + scDN.addServiceComponentHost(scDNH2); + scDNH2.persist(); + + Map<String, Set<String>> componentHostMap; + + componentHostMap = c1.getServiceComponentHostMap(null, null); + Assert.assertEquals(2, componentHostMap.size()); + + Assert.assertEquals(1, componentHostMap.get("NAMENODE").size()); + Assert.assertTrue(componentHostMap.get("NAMENODE").contains("h1")); + + Assert.assertEquals(2, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h1")); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h2")); + } + + @Test + public void testGetServiceComponentHostMap_ForService() throws Exception { + createDefaultCluster(); + + Service sfHDFS = serviceFactory.createNew(c1, "HDFS"); + c1.addService(sfHDFS); + sfHDFS.persist(); + + Service sfMR = serviceFactory.createNew(c1, "MAPREDUCE"); + c1.addService(sfMR); + sfMR.persist(); + + ServiceComponent scNN = serviceComponentFactory.createNew(sfHDFS, "NAMENODE"); + sfHDFS.addServiceComponent(scNN); + scNN.persist(); + ServiceComponentHost schNNH1 = serviceComponentHostFactory.createNew(scNN, "h1"); + scNN.addServiceComponentHost(schNNH1); + schNNH1.persist(); + + ServiceComponent scDN = serviceComponentFactory.createNew(sfHDFS, "DATANODE"); + sfHDFS.addServiceComponent(scDN); + scDN.persist(); + ServiceComponentHost scDNH1 = serviceComponentHostFactory.createNew(scDN, "h1"); + scDN.addServiceComponentHost(scDNH1); + scDNH1.persist(); + ServiceComponentHost scDNH2 = serviceComponentHostFactory.createNew(scDN, "h2"); + scDN.addServiceComponentHost(scDNH2); + scDNH2.persist(); + + ServiceComponent scJT = serviceComponentFactory.createNew(sfMR, "JOBTRACKER"); + sfMR.addServiceComponent(scJT); + scJT.persist(); + ServiceComponentHost schJTH1 = serviceComponentHostFactory.createNew(scJT, "h1"); + scJT.addServiceComponentHost(schJTH1); + schJTH1.persist(); + + Map<String, Set<String>> componentHostMap; + + componentHostMap = c1.getServiceComponentHostMap(null, Collections.singleton("HDFS")); + Assert.assertEquals(2, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("NAMENODE").size()); + Assert.assertTrue(componentHostMap.get("NAMENODE").contains("h1")); + Assert.assertEquals(2, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h1")); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h2")); + + componentHostMap = c1.getServiceComponentHostMap(null, Collections.singleton("MAPREDUCE")); + Assert.assertEquals(1, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("JOBTRACKER").size()); + Assert.assertTrue(componentHostMap.get("JOBTRACKER").contains("h1")); + + componentHostMap = c1.getServiceComponentHostMap(null, new HashSet<String>(Arrays.asList("HDFS", "MAPREDUCE"))); + Assert.assertEquals(3, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("NAMENODE").size()); + Assert.assertTrue(componentHostMap.get("NAMENODE").contains("h1")); + Assert.assertEquals(2, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h1")); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h2")); + Assert.assertEquals(1, componentHostMap.get("JOBTRACKER").size()); + Assert.assertTrue(componentHostMap.get("JOBTRACKER").contains("h1")); + + componentHostMap = c1.getServiceComponentHostMap(null, Collections.singleton("UNKNOWN")); + Assert.assertEquals(0, componentHostMap.size()); + } + + @Test + public void testGetServiceComponentHostMap_ForHost() throws Exception { + createDefaultCluster(); + + Service sfHDFS = serviceFactory.createNew(c1, "HDFS"); + c1.addService(sfHDFS); + sfHDFS.persist(); + + Service sfMR = serviceFactory.createNew(c1, "MAPREDUCE"); + c1.addService(sfMR); + sfMR.persist(); + + ServiceComponent scNN = serviceComponentFactory.createNew(sfHDFS, "NAMENODE"); + sfHDFS.addServiceComponent(scNN); + scNN.persist(); + ServiceComponentHost schNNH1 = serviceComponentHostFactory.createNew(scNN, "h1"); + scNN.addServiceComponentHost(schNNH1); + schNNH1.persist(); + + ServiceComponent scDN = serviceComponentFactory.createNew(sfHDFS, "DATANODE"); + sfHDFS.addServiceComponent(scDN); + scDN.persist(); + ServiceComponentHost scDNH1 = serviceComponentHostFactory.createNew(scDN, "h1"); + scDN.addServiceComponentHost(scDNH1); + scDNH1.persist(); + ServiceComponentHost scDNH2 = serviceComponentHostFactory.createNew(scDN, "h2"); + scDN.addServiceComponentHost(scDNH2); + scDNH2.persist(); + + ServiceComponent scJT = serviceComponentFactory.createNew(sfMR, "JOBTRACKER"); + sfMR.addServiceComponent(scJT); + scJT.persist(); + ServiceComponentHost schJTH1 = serviceComponentHostFactory.createNew(scJT, "h1"); + scJT.addServiceComponentHost(schJTH1); + schJTH1.persist(); + + Map<String, Set<String>> componentHostMap; + + componentHostMap = c1.getServiceComponentHostMap(Collections.singleton("h1"), null); + Assert.assertEquals(3, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("NAMENODE").size()); + Assert.assertTrue(componentHostMap.get("NAMENODE").contains("h1")); + Assert.assertEquals(1, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h1")); + Assert.assertEquals(1, componentHostMap.get("JOBTRACKER").size()); + Assert.assertTrue(componentHostMap.get("JOBTRACKER").contains("h1")); + + componentHostMap = c1.getServiceComponentHostMap(Collections.singleton("h2"), null); + Assert.assertEquals(1, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h2")); + + componentHostMap = c1.getServiceComponentHostMap(new HashSet<String>(Arrays.asList("h1", "h2", "h3")), null); + Assert.assertEquals(3, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("NAMENODE").size()); + Assert.assertTrue(componentHostMap.get("NAMENODE").contains("h1")); + Assert.assertEquals(2, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h1")); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h2")); + Assert.assertEquals(1, componentHostMap.get("JOBTRACKER").size()); + Assert.assertTrue(componentHostMap.get("JOBTRACKER").contains("h1")); + + componentHostMap = c1.getServiceComponentHostMap(Collections.singleton("unknown"), null); + Assert.assertEquals(0, componentHostMap.size()); + } + + @Test + public void testGetServiceComponentHostMap_ForHostAndService() throws Exception { + createDefaultCluster(); + + Service sfHDFS = serviceFactory.createNew(c1, "HDFS"); + c1.addService(sfHDFS); + sfHDFS.persist(); + + Service sfMR = serviceFactory.createNew(c1, "MAPREDUCE"); + c1.addService(sfMR); + sfMR.persist(); + + ServiceComponent scNN = serviceComponentFactory.createNew(sfHDFS, "NAMENODE"); + sfHDFS.addServiceComponent(scNN); + scNN.persist(); + ServiceComponentHost schNNH1 = serviceComponentHostFactory.createNew(scNN, "h1"); + scNN.addServiceComponentHost(schNNH1); + schNNH1.persist(); + + ServiceComponent scDN = serviceComponentFactory.createNew(sfHDFS, "DATANODE"); + sfHDFS.addServiceComponent(scDN); + scDN.persist(); + ServiceComponentHost scDNH1 = serviceComponentHostFactory.createNew(scDN, "h1"); + scDN.addServiceComponentHost(scDNH1); + scDNH1.persist(); + ServiceComponentHost scDNH2 = serviceComponentHostFactory.createNew(scDN, "h2"); + scDN.addServiceComponentHost(scDNH2); + scDNH2.persist(); + + ServiceComponent scJT = serviceComponentFactory.createNew(sfMR, "JOBTRACKER"); + sfMR.addServiceComponent(scJT); + scJT.persist(); + ServiceComponentHost schJTH1 = serviceComponentHostFactory.createNew(scJT, "h1"); + scJT.addServiceComponentHost(schJTH1); + schJTH1.persist(); + + Map<String, Set<String>> componentHostMap; + + componentHostMap = c1.getServiceComponentHostMap(Collections.singleton("h1"), Collections.singleton("HDFS")); + Assert.assertEquals(2, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h1")); + Assert.assertEquals(1, componentHostMap.get("NAMENODE").size()); + Assert.assertTrue(componentHostMap.get("NAMENODE").contains("h1")); + + componentHostMap = c1.getServiceComponentHostMap(Collections.singleton("h2"), Collections.singleton("HDFS")); + Assert.assertEquals(1, componentHostMap.size()); + Assert.assertEquals(1, componentHostMap.get("DATANODE").size()); + Assert.assertTrue(componentHostMap.get("DATANODE").contains("h2")); + + componentHostMap = c1.getServiceComponentHostMap(Collections.singleton("h3"), Collections.singleton("HDFS")); + Assert.assertEquals(0, componentHostMap.size()); + } + + @Test public void testGetAndSetConfigs() throws Exception { createDefaultCluster();
