Repository: ambari Updated Branches: refs/heads/branch-2.2 07b7aaead -> fec733436
AMBARI-14664. Blueprints: After Kerberos deployment, UI requires that Yarn service be restarted. (Sandor Magyari via rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fec73343 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fec73343 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fec73343 Branch: refs/heads/branch-2.2 Commit: fec733436551bfd2ca008502419d86c89ccb4380 Parents: 07b7aae Author: Bob Nettleton <rnettle...@hortonworks.com> Authored: Thu Jan 14 14:07:21 2016 -0500 Committer: Bob Nettleton <rnettle...@hortonworks.com> Committed: Thu Jan 14 14:07:21 2016 -0500 ---------------------------------------------------------------------- .../server/controller/KerberosHelperImpl.java | 18 +-- .../topology/ClusterConfigurationRequest.java | 119 +++++++++++++++++-- .../ClusterConfigurationRequestTest.java | 76 ++++++++++++ 3 files changed, 192 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/fec73343/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java index 0b586c9..07a6828 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java @@ -2104,18 +2104,20 @@ public class KerberosHelperImpl implements KerberosHelper { generalProperties.put("cluster_name", cluster.getClusterName()); // add clusterHostInfo config - Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster); + if (configurations.get("clusterHostInfo") == null) { + Map<String, Set<String>> clusterHostInfo = StageUtils.getClusterHostInfo(cluster); - if (clusterHostInfo != null) { - Map<String, String> componentHosts = new HashMap<String, String>(); + if (clusterHostInfo != null) { + Map<String, String> componentHosts = new HashMap<String, String>(); - clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); + clusterHostInfo = StageUtils.substituteHostIndexes(clusterHostInfo); - for (Map.Entry<String, Set<String>> entry : clusterHostInfo.entrySet()) { - componentHosts.put(entry.getKey(), StringUtils.join(entry.getValue(), ",")); - } + for (Map.Entry<String, Set<String>> entry : clusterHostInfo.entrySet()) { + componentHosts.put(entry.getKey(), StringUtils.join(entry.getValue(), ",")); + } - configurations.put("clusterHostInfo", componentHosts); + configurations.put("clusterHostInfo", componentHosts); + } } return configurations; http://git-wip-us.apache.org/repos/asf/ambari/blob/fec73343/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 9390b3a..16d983a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -29,6 +29,8 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.serveraction.kerberos.KerberosInvalidConfigurationException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.SecurityType; +import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +42,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Responsible for cluster configuration. @@ -48,6 +52,12 @@ public class ClusterConfigurationRequest { protected final static Logger LOG = LoggerFactory.getLogger(ClusterConfigurationRequest.class); + /** + * a regular expression Pattern used to find "clusterHostInfo.(component_name)_host" placeholders in strings + */ + private static final Pattern CLUSTER_HOST_INFO_PATTERN_VARIABLE = Pattern.compile("\\$\\{clusterHostInfo/?([\\w\\-\\.]+)_host(?:\\s*\\|\\s*(.+?))?\\}"); + public static final String CLUSTER_HOST_INFO = "clusterHostInfo"; + private AmbariContext ambariContext; private ClusterTopology clusterTopology; private BlueprintConfigurationProcessor configurationProcessor; @@ -76,25 +86,29 @@ public class ClusterConfigurationRequest { // get names of required host groups public Collection<String> getRequiredHostGroups() { - return configurationProcessor.getRequiredHostGroups(); + Collection<String> requiredHostGroups = new HashSet<String>(); + requiredHostGroups.addAll(configurationProcessor.getRequiredHostGroups()); + if (configureSecurity) { + requiredHostGroups.addAll(getRequiredHostgroupsForKerberosConfiguration()); + } + return requiredHostGroups; } public void process() throws AmbariException, ConfigurationTopologyException { // this will update the topo cluster config and all host group configs in the cluster topology - Set<String> updatedConfigTypes = Collections.emptySet(); + Set<String> updatedConfigTypes = new HashSet<>(); try { + if (configureSecurity) { + updatedConfigTypes.addAll(configureKerberos()); + } + // obtain recommended configurations before config updates if (!ConfigRecommendationStrategy.NEVER_APPLY.equals(this.clusterTopology.getConfigRecommendationStrategy())) { stackAdvisorBlueprintProcessor.adviseConfiguration(this.clusterTopology); } - if (configureSecurity) { - configureKerberos(); - } - - updatedConfigTypes = - configurationProcessor.doUpdateForClusterCreate(); + updatedConfigTypes.addAll(configurationProcessor.doUpdateForClusterCreate()); } catch (ConfigurationTopologyException e) { //log and continue to set configs on cluster to make progress LOG.error("An exception occurred while doing configuration topology update: " + e, e); @@ -103,20 +117,30 @@ public class ClusterConfigurationRequest { setConfigurationsOnCluster(clusterTopology, TopologyManager.TOPOLOGY_RESOLVED_TAG, updatedConfigTypes); } - private void configureKerberos() throws AmbariException { - String clusterName = ambariContext.getClusterName(clusterTopology.getClusterId()); - Cluster cluster = AmbariContext.getController().getClusters().getCluster(clusterName); + private Set<String> configureKerberos() throws AmbariException { + Set<String> updatedConfigTypes = new HashSet<>(); + + Cluster cluster = getCluster(); Blueprint blueprint = clusterTopology.getBlueprint(); + Configuration clusterConfiguration = clusterTopology.getConfiguration(); + Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties(); + // add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of + // StageUtils.getClusterInfo() + Map<String, String> componentHostsMap = createComponentHostMap(blueprint); + existingConfigurations.put("clusterHostInfo", componentHostsMap); try { + // generate principals & keytabs for headless identities AmbariContext.getController().getKerberosHelper() - .ensureHeadlessIdentities(cluster, clusterConfiguration.getFullProperties(), + .ensureHeadlessIdentities(cluster, existingConfigurations, new HashSet<String>(blueprint.getServices())); + // apply Kerberos specific configurations Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper() - .getServiceConfigurationUpdates(cluster, clusterConfiguration.getFullProperties(), + .getServiceConfigurationUpdates(cluster, existingConfigurations, new HashSet<String>(blueprint.getServices()), false); + for (String configType : updatedConfigs.keySet()) { Map<String, String> propertyMap = updatedConfigs.get(configType); for (String property : propertyMap.keySet()) { @@ -125,10 +149,79 @@ public class ClusterConfigurationRequest { clusterConfiguration.setProperty(configType, property, propertyMap.get(property)); } } + updatedConfigTypes.addAll(updatedConfigs.keySet()); + } catch (KerberosInvalidConfigurationException e) { + LOG.error("An exception occurred while doing Kerberos related configuration update: " + e, e); + } + + return updatedConfigTypes; + } + + private Map<String, String> createComponentHostMap(Blueprint blueprint) { + Map<String, String> componentHostsMap = new HashMap<String, String>(); + for (String service : blueprint.getServices()) { + Collection<String> components = blueprint.getComponents(service); + for (String component : components) { + Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component); + // retrieve corresponding clusterInfoKey for component using StageUtils + String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component); + if (clusterInfoKey == null) { + clusterInfoKey = component.toLowerCase() + "_hosts"; + } + componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); + } + } + return componentHostsMap; + } + + private Collection<String> getRequiredHostgroupsForKerberosConfiguration() { + Collection<String> requiredHostGroups = new HashSet<String>(); + + try { + Cluster cluster = getCluster(); + Blueprint blueprint = clusterTopology.getBlueprint(); + + Configuration clusterConfiguration = clusterTopology.getConfiguration(); + Map<String, Map<String, String>> existingConfigurations = clusterConfiguration.getFullProperties(); + existingConfigurations.put(CLUSTER_HOST_INFO, new HashMap<String, String>()); + + // apply Kerberos specific configurations + Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper() + .getServiceConfigurationUpdates(cluster, existingConfigurations, + new HashSet<String>(blueprint.getServices()), false); + + // retrieve hostgroup for component names extracted from variables like "{clusterHostInfo.(component_name) + // _host}" + for (String configType : updatedConfigs.keySet()) { + Map<String, String> propertyMap = updatedConfigs.get(configType); + for (String property : propertyMap.keySet()) { + String propertyValue = propertyMap.get(property); + Matcher matcher = CLUSTER_HOST_INFO_PATTERN_VARIABLE.matcher(propertyValue); + while (matcher.find()) { + String component = matcher.group(1).toUpperCase(); + Collection<String> hostGroups = clusterTopology.getHostGroupsForComponent(component); + if (hostGroups.isEmpty()) { + LOG.warn("No matching hostgroup found for component: {} specified in Kerberos config type: {} property:" + + " " + + "{}", component, configType, property); + } else { + requiredHostGroups.addAll(hostGroups); + } + } + } + } } catch (KerberosInvalidConfigurationException e) { LOG.error("An exception occurred while doing Kerberos related configuration update: " + e, e); + } catch (AmbariException e) { + LOG.error("An exception occurred while doing Kerberos related configuration update: " + e, e); } + return requiredHostGroups; + } + + private Cluster getCluster() throws AmbariException { + String clusterName = ambariContext.getClusterName(clusterTopology.getClusterId()); + return AmbariContext.getController().getClusters().getCluster(clusterName); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/fec73343/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index 26ab63f..96abe9d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -35,9 +35,11 @@ import org.powermock.api.easymock.PowerMock; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -114,6 +116,18 @@ public class ClusterConfigurationRequestTest { services.add("KERBEROS"); services.add("ZOOKEPER"); expect(blueprint.getServices()).andReturn(services).anyTimes(); + + List<String> hdfsComponents = new ArrayList<>(); + hdfsComponents.add("NAMENODE"); + List<String> kerberosComponents = new ArrayList<>(); + kerberosComponents.add("KERBEROS_CLIENT"); + List<String> zookeeperComponents = new ArrayList<>(); + zookeeperComponents.add("ZOOKEEPER_SERVER"); + + expect(blueprint.getComponents("HDFS")).andReturn(hdfsComponents).anyTimes(); + expect(blueprint.getComponents("KERBEROS")).andReturn(kerberosComponents).anyTimes(); + expect(blueprint.getComponents("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes(); @@ -145,4 +159,66 @@ public class ClusterConfigurationRequestTest { } + @Test + public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Exception { + + Map<String, Map<String, String>> existingConfig = new HashMap<String, Map<String, String>>(); + Configuration stackConfig = new Configuration(existingConfig, + new HashMap<String, Map<String, Map<String, String>>>()); + + PowerMock.mockStatic(AmbariContext.class); + AmbariContext.getController(); + expectLastCall().andReturn(controller).anyTimes(); + + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); + + expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.<String>singletonList("testConfigType") + ).anyTimes(); + expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.<String>emptySet()).anyTimes(); + expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.<String, + Stack.ConfigProperty>emptyMap()).anyTimes(); + + Set<String> services = new HashSet<>(); + services.add("HDFS"); + services.add("KERBEROS"); + services.add("ZOOKEPER"); + expect(blueprint.getServices()).andReturn(services).anyTimes(); + + List<String> hdfsComponents = new ArrayList<>(); + hdfsComponents.add("NAMENODE"); + List<String> kerberosComponents = new ArrayList<>(); + kerberosComponents.add("KERBEROS_CLIENT"); + List<String> zookeeperComponents = new ArrayList<>(); + zookeeperComponents.add("ZOOKEEPER_SERVER"); + + expect(blueprint.getComponents("HDFS")).andReturn(hdfsComponents).anyTimes(); + expect(blueprint.getComponents("KERBEROS")).andReturn(kerberosComponents).anyTimes(); + expect(blueprint.getComponents("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + + expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes(); + expect(topology.getHostGroupInfo()).andReturn(Collections.<String, HostGroupInfo>emptyMap()); + expect(topology.getClusterId()).andReturn(Long.valueOf(1)).anyTimes(); + expect(ambariContext.getClusterName(Long.valueOf(1))).andReturn("testCluster").anyTimes(); + expect(ambariContext.createConfigurationRequests(anyObject(Map.class))).andReturn(Collections + .<ConfigurationRequest>emptyList()).anyTimes(); + + + PowerMock.replay(stack, blueprint, topology, controller, clusters, ambariContext, + AmbariContext + .class); + + ClusterConfigurationRequest clusterConfigurationRequest = new ClusterConfigurationRequest( + ambariContext, topology, false, stackAdvisorBlueprintProcessor); + clusterConfigurationRequest.process(); + + verify(blueprint, topology, ambariContext, controller); + + } + + + }
