Repository: ambari Updated Branches: refs/heads/trunk 27a9d78c9 -> c19b78966
AMBARI-7738. Fix blueprint processor to properly handle config topology update for components with a valid cardinality of 0 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c19b7896 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c19b7896 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c19b7896 Branch: refs/heads/trunk Commit: c19b7896667bbeedf83525b7b7c7e62e31a96b57 Parents: 27a9d78 Author: Robert Nettleton <[email protected]> Authored: Thu Oct 23 14:44:55 2014 -0400 Committer: John Speidel <[email protected]> Committed: Thu Oct 23 14:45:51 2014 -0400 ---------------------------------------------------------------------- .../internal/BaseBlueprintProcessor.java | 487 ------------------- .../BlueprintConfigurationProcessor.java | 89 ++-- .../server/controller/internal/Cardinality.java | 86 ++++ .../internal/ClusterResourceProvider.java | 4 +- .../server/controller/internal/Stack.java | 450 +++++++++++++++++ .../internal/BaseBlueprintProcessorTest.java | 24 +- .../BlueprintConfigurationProcessorTest.java | 211 +++++++- .../internal/ClusterResourceProviderTest.java | 24 +- 8 files changed, 812 insertions(+), 563 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java index abd22a4..9be73cc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessor.java @@ -24,13 +24,6 @@ import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.StackConfigurationRequest; -import org.apache.ambari.server.controller.StackConfigurationResponse; -import org.apache.ambari.server.controller.StackLevelConfigurationRequest; -import org.apache.ambari.server.controller.StackServiceComponentRequest; -import org.apache.ambari.server.controller.StackServiceComponentResponse; -import org.apache.ambari.server.controller.StackServiceRequest; -import org.apache.ambari.server.controller.StackServiceResponse; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.orm.dao.BlueprintDAO; @@ -42,10 +35,8 @@ import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DependencyInfo; -import org.apache.ambari.server.state.PropertyInfo; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -455,418 +446,6 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP // ----- Inner Classes ----------------------------------------------------- /** - * Encapsulates stack information. - */ - protected static class Stack { - /** - * Stack name - */ - private String name; - - /** - * Stack version - */ - private String version; - - /** - * Map of service name to components - */ - private Map<String, Collection<String>> serviceComponents = - new HashMap<String, Collection<String>>(); - - /** - * Map of component to service - */ - private Map<String, String> componentService = new HashMap<String, String>(); - - /** - * Map of component to dependencies - */ - private Map<String, Collection<DependencyInfo>> dependencies = - new HashMap<String, Collection<DependencyInfo>>(); - - /** - * Map of dependency to conditional service - */ - private Map<DependencyInfo, String> dependencyConditionalServiceMap = - new HashMap<DependencyInfo, String>(); - - /** - * Map of database component name to configuration property which indicates whether - * the database in to be managed or if it is an external non-managed instance. - * If the value of the config property starts with 'New', the database is determined - * to be managed, otherwise it is non-managed. - */ - private Map<String, String> dbDependencyInfo = new HashMap<String, String>(); - - /** - * Map of component to required cardinality - */ - private Map<String, String> cardinalityRequirements = new HashMap<String, String>(); - - /** - * Map of component to auto-deploy information - */ - private Map<String, AutoDeployInfo> componentAutoDeployInfo = - new HashMap<String, AutoDeployInfo>(); - - /** - * Map of service to config type properties - */ - private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations = - new HashMap<String, Map<String, Map<String, ConfigProperty>>>(); - - - /** - * Ambari Management Controller, used to obtain Stack definitions - */ - private final AmbariManagementController ambariManagementController; - - /** - * Contains a configuration property's value and attributes. - */ - private class ConfigProperty { - - private ConfigProperty(String value, Map<String, String> attributes) { - this.value = value; - this.attributes = attributes; - } - - private String value; - private Map<String, String> attributes; - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public Map<String, String> getAttributes() { - return attributes; - } - - public void setAttributes(Map<String, String> attributes) { - this.attributes = attributes; - } - } - - /** - * Constructor. - * - * @param name stack name - * @param version stack version - * - * @throws AmbariException an exception occurred getting stack information - * for the specified name and version - */ - public Stack(String name, String version, AmbariManagementController ambariManagementController) throws AmbariException { - this.name = name; - this.version = version; - this.ambariManagementController = ambariManagementController; - - Set<StackServiceResponse> stackServices = ambariManagementController.getStackServices( - Collections.singleton(new StackServiceRequest(name, version, null))); - - for (StackServiceResponse stackService : stackServices) { - String serviceName = stackService.getServiceName(); - parseComponents(serviceName); - parseConfigurations(serviceName); - registerConditionalDependencies(); - } - } - - /** - * Obtain stack name. - * - * @return stack name - */ - public String getName() { - return name; - } - - /** - * Obtain stack version. - * - * @return stack version - */ - public String getVersion() { - return version; - } - - - Map<DependencyInfo, String> getDependencyConditionalServiceMap() { - return dependencyConditionalServiceMap; - } - - /** - * Get services contained in the stack. - * - * @return collection of all services for the stack - */ - public Collection<String> getServices() { - return serviceComponents.keySet(); - } - - /** - * Get components contained in the stack for the specified service. - * - * @param service service name - * - * @return collection of component names for the specified service - */ - public Collection<String> getComponents(String service) { - return serviceComponents.get(service); - } - - /** - * Get configuration types for the specified service. - * - * @param service service name - * - * @return collection of configuration types for the specified service - */ - public Collection<String> getConfigurationTypes(String service) { - return serviceConfigurations.get(service).keySet(); - } - - /** - * Get config properties for the specified service and configuration type. - * - * @param service service name - * @param type configuration type - * - * @return map of property names to values for the specified service and configuration type - */ - public Map<String, String> getConfigurationProperties(String service, String type) { - Map<String, String> configMap = new HashMap<String, String>(); - Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); - if (configProperties != null) { - for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { - configMap.put(configProperty.getKey(), configProperty.getValue().getValue()); - } - } - return configMap; - } - - /** - * Get config attributes for the specified service and configuration type. - * - * @param service service name - * @param type configuration type - * - * @return map of attribute names to map of property names to attribute values - * for the specified service and configuration type - */ - public Map<String, Map<String, String>> getConfigurationAttributes(String service, String type) { - Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>(); - Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); - if (configProperties != null) { - for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { - String propertyName = configProperty.getKey(); - Map<String, String> propertyAttributes = configProperty.getValue().getAttributes(); - if (propertyAttributes != null) { - for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) { - String attributeName = propertyAttribute.getKey(); - String attributeValue = propertyAttribute.getValue(); - Map<String, String> attributes = attributesMap.get(attributeName); - if (attributes == null) { - attributes = new HashMap<String, String>(); - attributesMap.put(attributeName, attributes); - } - attributes.put(propertyName, attributeValue); - } - } - } - } - return attributesMap; - } - - /** - * Get the service for the specified component. - * - * @param component component name - * - * @return service name that contains tha specified component - */ - public String getServiceForComponent(String component) { - return componentService.get(component); - } - - /** - * Get the names of the services which contains the specified components. - * - * @param components collection of components - * - * @return collection of services which contain the specified components - */ - public Collection<String> getServicesForComponents(Collection<String> components) { - Set<String> services = new HashSet<String>(); - for (String component : components) { - services.add(getServiceForComponent(component)); - } - - return services; - } - - /** - * Obtain the service name which corresponds to the specified configuration. - * - * @param config configuration type - * - * @return name of service which corresponds to the specified configuration type - */ - public String getServiceForConfigType(String config) { - for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) { - Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue(); - if (typeMap.containsKey(config)) { - return entry.getKey(); - } - } - throw new IllegalArgumentException( - "Specified configuration type is not associated with any service: " + config); - } - - /** - * Return the dependencies specified for the given component. - * - * @param component component to get dependency information for - * - * @return collection of dependency information for the specified component - */ - //todo: full dependency graph - public Collection<DependencyInfo> getDependenciesForComponent(String component) { - return dependencies.containsKey(component) ? dependencies.get(component) : - Collections.<DependencyInfo>emptySet(); - } - - /** - * Get the service, if any, that a component dependency is conditional on. - * - * @param dependency dependency to get conditional service for - * - * @return conditional service for provided component or null if dependency - * is not conditional on a service - */ - public String getConditionalServiceForDependency(DependencyInfo dependency) { - return dependencyConditionalServiceMap.get(dependency); - } - - public String getExternalComponentConfig(String component) { - return dbDependencyInfo.get(component); - } - - /** - * Obtain the required cardinality for the specified component. - */ - public Cardinality getCardinality(String component) { - return new Cardinality(cardinalityRequirements.get(component)); - } - - /** - * Obtain auto-deploy information for the specified component. - */ - public AutoDeployInfo getAutoDeployInfo(String component) { - return componentAutoDeployInfo.get(component); - } - - /** - * Parse components for the specified service from the stack definition. - * - * @param service service name - * - * @throws AmbariException an exception occurred getting components from the stack definition - */ - private void parseComponents(String service) throws AmbariException{ - Collection<String> componentSet = new HashSet<String>(); - - Set<StackServiceComponentResponse> components = ambariManagementController.getStackComponents( - Collections.singleton(new StackServiceComponentRequest(name, version, service, null))); - - // stack service components - for (StackServiceComponentResponse component : components) { - String componentName = component.getComponentName(); - componentSet.add(componentName); - componentService.put(componentName, service); - String cardinality = component.getCardinality(); - if (cardinality != null) { - cardinalityRequirements.put(componentName, cardinality); - } - AutoDeployInfo autoDeploy = component.getAutoDeploy(); - if (autoDeploy != null) { - componentAutoDeployInfo.put(componentName, autoDeploy); - } - - // populate component dependencies - Collection<DependencyInfo> componentDependencies = stackInfo.getComponentDependencies( - name, version, service, componentName); - - if (componentDependencies != null && ! componentDependencies.isEmpty()) { - dependencies.put(componentName, componentDependencies); - } - } - this.serviceComponents.put(service, componentSet); - } - - /** - * Parse configurations for the specified service from the stack definition. - * - * @param service service name - * - * @throws AmbariException an exception occurred getting configurations from the stack definition - */ - private void parseConfigurations(String service) throws AmbariException { - Map<String, Map<String, ConfigProperty>> mapServiceConfig = new HashMap<String, Map<String, ConfigProperty>>(); - - serviceConfigurations.put(service, mapServiceConfig); - - Set<StackConfigurationResponse> serviceConfigs = ambariManagementController.getStackConfigurations( - Collections.singleton(new StackConfigurationRequest(name, version, service, null))); - Set<StackConfigurationResponse> stackLevelConfigs = ambariManagementController.getStackLevelConfigurations( - Collections.singleton(new StackLevelConfigurationRequest(name, version, null))); - serviceConfigs.addAll(stackLevelConfigs); - - for (StackConfigurationResponse config : serviceConfigs) { - String type = config.getType(); - //strip .xml from type - if (type.endsWith(".xml")) { - type = type.substring(0, type.length() - 4); - } - Map<String, ConfigProperty> mapTypeConfig = mapServiceConfig.get(type); - if (mapTypeConfig == null) { - mapTypeConfig = new HashMap<String, ConfigProperty>(); - mapServiceConfig.put(type, mapTypeConfig); - } - mapTypeConfig.put(config.getPropertyName(), - new ConfigProperty(config.getPropertyValue(), config.getPropertyAttributes())); - } - } - - /** - * Register conditional dependencies. - */ - //todo: This information should be specified in the stack definition. - void registerConditionalDependencies() { - Collection<DependencyInfo> nagiosDependencies = getDependenciesForComponent("NAGIOS_SERVER"); - for (DependencyInfo dependency : nagiosDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database"); - } - } - - /** * Host group representation. */ protected static class HostGroupImpl implements HostGroup { @@ -1094,70 +673,4 @@ public abstract class BaseBlueprintProcessor extends AbstractControllerResourceP } } - /** - * Component cardinality representation. - */ - protected static class Cardinality { - String cardinality; - int min = 0; - int max = Integer.MAX_VALUE; - int exact = -1; - boolean isAll = false; - - public Cardinality(String cardinality) { - this.cardinality = cardinality; - if (cardinality != null && ! cardinality.isEmpty()) { - if (cardinality.contains("+")) { - min = Integer.valueOf(cardinality.split("\\+")[0]); - } else if (cardinality.contains("-")) { - String[] toks = cardinality.split("-"); - min = Integer.parseInt(toks[0]); - max = Integer.parseInt(toks[1]); - } else if (cardinality.equals("ALL")) { - isAll = true; - } else { - exact = Integer.parseInt(cardinality); - } - } - } - - /** - * Determine if component is required for all host groups. - * - * @return true if cardinality is 'ALL', false otherwise - */ - public boolean isAll() { - return isAll; - } - - /** - * Determine if the given count satisfies the required cardinality. - * - * @param count number of host groups containing component - * - * @return true id count satisfies the required cardinality, false otherwise - */ - public boolean isValidCount(int count) { - if (isAll) { - return false; - } else if (exact != -1) { - return count == exact; - } else return count >= min && count <= max; - } - - /** - * Determine if the cardinality count supports auto-deployment. - * This determination is independent of whether the component is configured - * to be auto-deployed. This only indicates whether auto-deployment is - * supported for the current cardinality. - * - * At this time, only cardinalities of ALL or where a count of 1 is valid are - * supported. - * - * @return true if cardinality supports auto-deployment - */ - public boolean supportsAutoDeploy() { - return isValidCount(1) || isAll; - } - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 10cc016..4183b9d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.controller.internal; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -112,11 +113,12 @@ public class BlueprintConfigurationProcessor { * Update properties for cluster creation. This involves updating topology related properties with * concrete topology information. * - * @param hostGroups host groups of cluster to be deployed + * @param hostGroups host groups of cluster to be deployed + * @param stackDefinition stack used for cluster creation * * @return updated properties */ - public Map<String, Map<String, String>> doUpdateForClusterCreate(Map<String, ? extends HostGroup> hostGroups) { + public Map<String, Map<String, String>> doUpdateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, Stack stackDefinition) { for (Map<String, Map<String, PropertyUpdater>> updaterMap : createCollectionOfUpdaters()) { for (Map.Entry<String, Map<String, PropertyUpdater>> entry : updaterMap.entrySet()) { String type = entry.getKey(); @@ -127,7 +129,7 @@ public class BlueprintConfigurationProcessor { Map<String, String> typeMap = properties.get(type); if (typeMap != null && typeMap.containsKey(propertyName)) { typeMap.put(propertyName, updater.updateForClusterCreate( - hostGroups, typeMap.get(propertyName), properties)); + hostGroups, typeMap.get(propertyName), properties, stackDefinition)); } } } @@ -490,14 +492,17 @@ public class BlueprintConfigurationProcessor { * Update a property value. * * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties + * @param hostGroups host groups + * @param origValue original value of property + * @param properties all properties + * @param stackDefinition definition of stack used for this cluster + * creation attempt * * @return new property value */ public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String origValue, Map<String, Map<String, String>> properties); + String origValue, Map<String, Map<String, String>> properties, Stack stackDefinition + ); } /** @@ -523,15 +528,17 @@ public class BlueprintConfigurationProcessor { * Update the property with the new host name which runs the associated component. * * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties + * @param hostGroups host groups + * @param origValue original value of property + * @param properties all properties + * @param stackDefinition stack used for cluster creation * * @return updated property value with old host name replaced by new host name */ public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, - Map<String, Map<String, String>> properties) { + Map<String, Map<String, String>> properties, + Stack stackDefinition) { Matcher m = HOSTGROUP_REGEX.matcher(origValue); if (m.find()) { @@ -544,8 +551,18 @@ public class BlueprintConfigurationProcessor { if (matchingGroups.size() == 1) { return origValue.replace("localhost", matchingGroups.iterator().next().getHostInfo().iterator().next()); } else { - throw new IllegalArgumentException("Unable to update configuration property with topology information. " + + Cardinality cardinality = stackDefinition.getCardinality(component); + // if no matching host groups are found for a component whose configuration + // is handled by this updater, check the stack first to determine if + // zero is a valid cardinality for this component. This is necessary + // in the case of a component in "technical preview" status, since it + // may be valid to have 0 or 1 instances of such a component in the cluster + if (matchingGroups.isEmpty() && cardinality.isValidCount(0)) { + return origValue; + } else { + throw new IllegalArgumentException("Unable to update configuration property with topology information. " + "Component '" + this.component + "' is not mapped to any host group or is mapped to multiple groups."); + } } } } @@ -597,19 +614,21 @@ public class BlueprintConfigurationProcessor { * original value. * * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties + * @param hostGroups host groups + * @param origValue original value of property + * @param properties all properties + * @param stackDefinition stack used for cluster creation * * @return updated property value with old host name replaced by new host name or original value * if the database is external */ @Override public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, - String origValue, Map<String, Map<String, String>> properties) { + String origValue, Map<String, Map<String, String>> properties, + Stack stackDefinition) { if (isDatabaseManaged(properties)) { - return super.updateForClusterCreate(hostGroups, origValue, properties); + return super.updateForClusterCreate(hostGroups, origValue, properties, stackDefinition); } else { return origValue; } @@ -656,15 +675,17 @@ public class BlueprintConfigurationProcessor { * component. * * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties + * @param hostGroups host groups + * @param origValue original value of property + * @param properties all properties + * @param stackDefinition stack used for cluster creation * * @return updated property value with old host names replaced by new host names */ public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, - Map<String, Map<String, String>> properties) { + Map<String, Map<String, String>> properties, + Stack stackDefinition) { Collection<String> hostStrings = getHostStrings(hostGroups, origValue); if (hostStrings.isEmpty()) { @@ -709,15 +730,17 @@ public class BlueprintConfigurationProcessor { * Append 'm' to the original property value if it doesn't already exist. * * - * @param hostGroups host groups - * @param origValue original value of property - * @param properties all properties + * @param hostGroups host groups + * @param origValue original value of property + * @param properties all properties + * @param stackDefinition stack used for cluster creation * * @return property with 'm' appended */ public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, Map<String, - Map<String, String>> properties) { + Map<String, String>> properties, + Stack stackDefinition) { return origValue.endsWith("m") ? origValue : origValue + 'm'; } @@ -741,18 +764,20 @@ public class BlueprintConfigurationProcessor { /** * Return decorated form of the updated input property value. * - * @param hostGroupMap map of host group name to HostGroup - * @param origValue original value of property - * @param properties all properties + * @param hostGroupMap map of host group name to HostGroup + * @param origValue original value of property + * @param properties all properties + * @param stackDefinition stack used for cluster creation * * @return Formatted output string */ @Override public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroupMap, String origValue, - Map<String, Map<String, String>> properties) { + Map<String, Map<String, String>> properties, + Stack stackDefinition) { - return doFormat(propertyUpdater.updateForClusterCreate(hostGroupMap, origValue, properties)); + return doFormat(propertyUpdater.updateForClusterCreate(hostGroupMap, origValue, properties, stackDefinition)); } /** @@ -812,7 +837,9 @@ public class BlueprintConfigurationProcessor { */ private static class OriginalValuePropertyUpdater implements PropertyUpdater { @Override - public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, Map<String, Map<String, String>> properties) { + public String updateForClusterCreate(Map<String, ? extends HostGroup> hostGroups, String origValue, + Map<String, Map<String, String>> properties, + Stack stackDefinition) { // always return the original value, since these properties do not require update handling return origValue; } http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Cardinality.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Cardinality.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Cardinality.java new file mode 100644 index 0000000..74e594f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Cardinality.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +/** + * Component cardinality representation. + */ +class Cardinality { + String cardinality; + int min = 0; + int max = Integer.MAX_VALUE; + int exact = -1; + boolean isAll = false; + + public Cardinality(String cardinality) { + this.cardinality = cardinality; + if (cardinality != null && ! cardinality.isEmpty()) { + if (cardinality.contains("+")) { + min = Integer.valueOf(cardinality.split("\\+")[0]); + } else if (cardinality.contains("-")) { + String[] toks = cardinality.split("-"); + min = Integer.parseInt(toks[0]); + max = Integer.parseInt(toks[1]); + } else if (cardinality.equals("ALL")) { + isAll = true; + } else { + exact = Integer.parseInt(cardinality); + } + } + } + + /** + * Determine if component is required for all host groups. + * + * @return true if cardinality is 'ALL', false otherwise + */ + public boolean isAll() { + return isAll; + } + + /** + * Determine if the given count satisfies the required cardinality. + * + * @param count number of host groups containing component + * + * @return true id count satisfies the required cardinality, false otherwise + */ + public boolean isValidCount(int count) { + if (isAll) { + return false; + } else if (exact != -1) { + return count == exact; + } else return count >= min && count <= max; + } + + /** + * Determine if the cardinality count supports auto-deployment. + * This determination is independent of whether the component is configured + * to be auto-deployed. This only indicates whether auto-deployment is + * supported for the current cardinality. + * + * At this time, only cardinalities of ALL or where a count of 1 is valid are + * supported. + * + * @return true if cardinality supports auto-deployment + */ + public boolean supportsAutoDeploy() { + return isValidCount(1) || isAll; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java index 090eb92..8dd06ec 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java @@ -29,7 +29,6 @@ import java.util.Set; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.PersistKeyValueService; import org.apache.ambari.server.controller.*; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.NoSuchResourceException; @@ -49,7 +48,6 @@ import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.ConfigImpl; -import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.StackId; /** @@ -879,7 +877,7 @@ public class ClusterResourceProvider extends BaseBlueprintProcessor { processBlueprintClusterConfigAttributes(blueprintAttributes); BlueprintConfigurationProcessor configurationProcessor = new BlueprintConfigurationProcessor(mapClusterConfigurations); - configurationProcessor.doUpdateForClusterCreate(blueprintHostGroups); + configurationProcessor.doUpdateForClusterCreate(blueprintHostGroups, stack); setMissingConfigurations(blueprintHostGroups); } http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java new file mode 100644 index 0000000..3ccae43 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -0,0 +1,450 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.StackConfigurationRequest; +import org.apache.ambari.server.controller.StackConfigurationResponse; +import org.apache.ambari.server.controller.StackLevelConfigurationRequest; +import org.apache.ambari.server.controller.StackServiceComponentRequest; +import org.apache.ambari.server.controller.StackServiceComponentResponse; +import org.apache.ambari.server.controller.StackServiceRequest; +import org.apache.ambari.server.controller.StackServiceResponse; +import org.apache.ambari.server.state.AutoDeployInfo; +import org.apache.ambari.server.state.DependencyInfo; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Encapsulates stack information. + */ +class Stack { + /** + * Stack name + */ + private String name; + + /** + * Stack version + */ + private String version; + + /** + * Map of service name to components + */ + private Map<String, Collection<String>> serviceComponents = + new HashMap<String, Collection<String>>(); + + /** + * Map of component to service + */ + private Map<String, String> componentService = new HashMap<String, String>(); + + /** + * Map of component to dependencies + */ + private Map<String, Collection<DependencyInfo>> dependencies = + new HashMap<String, Collection<DependencyInfo>>(); + + /** + * Map of dependency to conditional service + */ + private Map<DependencyInfo, String> dependencyConditionalServiceMap = + new HashMap<DependencyInfo, String>(); + + /** + * Map of database component name to configuration property which indicates whether + * the database in to be managed or if it is an external non-managed instance. + * If the value of the config property starts with 'New', the database is determined + * to be managed, otherwise it is non-managed. + */ + private Map<String, String> dbDependencyInfo = new HashMap<String, String>(); + + /** + * Map of component to required cardinality + */ + private Map<String, String> cardinalityRequirements = new HashMap<String, String>(); + + /** + * Map of component to auto-deploy information + */ + private Map<String, AutoDeployInfo> componentAutoDeployInfo = + new HashMap<String, AutoDeployInfo>(); + + /** + * Map of service to config type properties + */ + private Map<String, Map<String, Map<String, ConfigProperty>>> serviceConfigurations = + new HashMap<String, Map<String, Map<String, ConfigProperty>>>(); + + + /** + * Ambari Management Controller, used to obtain Stack definitions + */ + private final AmbariManagementController ambariManagementController; + + /** + * Contains a configuration property's value and attributes. + */ + private class ConfigProperty { + + private ConfigProperty(String value, Map<String, String> attributes) { + this.value = value; + this.attributes = attributes; + } + + private String value; + private Map<String, String> attributes; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public Map<String, String> getAttributes() { + return attributes; + } + + public void setAttributes(Map<String, String> attributes) { + this.attributes = attributes; + } + } + + /** + * Constructor. + * + * @param name stack name + * @param version stack version + * + * @throws org.apache.ambari.server.AmbariException an exception occurred getting stack information + * for the specified name and version + */ + public Stack(String name, String version, AmbariManagementController ambariManagementController) throws AmbariException { + this.name = name; + this.version = version; + this.ambariManagementController = ambariManagementController; + + Set<StackServiceResponse> stackServices = ambariManagementController.getStackServices( + Collections.singleton(new StackServiceRequest(name, version, null))); + + for (StackServiceResponse stackService : stackServices) { + String serviceName = stackService.getServiceName(); + parseComponents(serviceName); + parseConfigurations(serviceName); + registerConditionalDependencies(); + } + } + + /** + * Obtain stack name. + * + * @return stack name + */ + public String getName() { + return name; + } + + /** + * Obtain stack version. + * + * @return stack version + */ + public String getVersion() { + return version; + } + + + Map<DependencyInfo, String> getDependencyConditionalServiceMap() { + return dependencyConditionalServiceMap; + } + + /** + * Get services contained in the stack. + * + * @return collection of all services for the stack + */ + public Collection<String> getServices() { + return serviceComponents.keySet(); + } + + /** + * Get components contained in the stack for the specified service. + * + * @param service service name + * + * @return collection of component names for the specified service + */ + public Collection<String> getComponents(String service) { + return serviceComponents.get(service); + } + + /** + * Get configuration types for the specified service. + * + * @param service service name + * + * @return collection of configuration types for the specified service + */ + public Collection<String> getConfigurationTypes(String service) { + return serviceConfigurations.get(service).keySet(); + } + + /** + * Get config properties for the specified service and configuration type. + * + * @param service service name + * @param type configuration type + * + * @return map of property names to values for the specified service and configuration type + */ + public Map<String, String> getConfigurationProperties(String service, String type) { + Map<String, String> configMap = new HashMap<String, String>(); + Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + configMap.put(configProperty.getKey(), configProperty.getValue().getValue()); + } + } + return configMap; + } + + /** + * Get config attributes for the specified service and configuration type. + * + * @param service service name + * @param type configuration type + * + * @return map of attribute names to map of property names to attribute values + * for the specified service and configuration type + */ + public Map<String, Map<String, String>> getConfigurationAttributes(String service, String type) { + Map<String, Map<String, String>> attributesMap = new HashMap<String, Map<String, String>>(); + Map<String, ConfigProperty> configProperties = serviceConfigurations.get(service).get(type); + if (configProperties != null) { + for (Map.Entry<String, ConfigProperty> configProperty : configProperties.entrySet()) { + String propertyName = configProperty.getKey(); + Map<String, String> propertyAttributes = configProperty.getValue().getAttributes(); + if (propertyAttributes != null) { + for (Map.Entry<String, String> propertyAttribute : propertyAttributes.entrySet()) { + String attributeName = propertyAttribute.getKey(); + String attributeValue = propertyAttribute.getValue(); + Map<String, String> attributes = attributesMap.get(attributeName); + if (attributes == null) { + attributes = new HashMap<String, String>(); + attributesMap.put(attributeName, attributes); + } + attributes.put(propertyName, attributeValue); + } + } + } + } + return attributesMap; + } + + /** + * Get the service for the specified component. + * + * @param component component name + * + * @return service name that contains tha specified component + */ + public String getServiceForComponent(String component) { + return componentService.get(component); + } + + /** + * Get the names of the services which contains the specified components. + * + * @param components collection of components + * + * @return collection of services which contain the specified components + */ + public Collection<String> getServicesForComponents(Collection<String> components) { + Set<String> services = new HashSet<String>(); + for (String component : components) { + services.add(getServiceForComponent(component)); + } + + return services; + } + + /** + * Obtain the service name which corresponds to the specified configuration. + * + * @param config configuration type + * + * @return name of service which corresponds to the specified configuration type + */ + public String getServiceForConfigType(String config) { + for (Map.Entry<String, Map<String, Map<String, ConfigProperty>>> entry : serviceConfigurations.entrySet()) { + Map<String, Map<String, ConfigProperty>> typeMap = entry.getValue(); + if (typeMap.containsKey(config)) { + return entry.getKey(); + } + } + throw new IllegalArgumentException( + "Specified configuration type is not associated with any service: " + config); + } + + /** + * Return the dependencies specified for the given component. + * + * @param component component to get dependency information for + * + * @return collection of dependency information for the specified component + */ + //todo: full dependency graph + public Collection<DependencyInfo> getDependenciesForComponent(String component) { + return dependencies.containsKey(component) ? dependencies.get(component) : + Collections.<DependencyInfo>emptySet(); + } + + /** + * Get the service, if any, that a component dependency is conditional on. + * + * @param dependency dependency to get conditional service for + * + * @return conditional service for provided component or null if dependency + * is not conditional on a service + */ + public String getConditionalServiceForDependency(DependencyInfo dependency) { + return dependencyConditionalServiceMap.get(dependency); + } + + public String getExternalComponentConfig(String component) { + return dbDependencyInfo.get(component); + } + + /** + * Obtain the required cardinality for the specified component. + */ + public Cardinality getCardinality(String component) { + return new Cardinality(cardinalityRequirements.get(component)); + } + + /** + * Obtain auto-deploy information for the specified component. + */ + public AutoDeployInfo getAutoDeployInfo(String component) { + return componentAutoDeployInfo.get(component); + } + + /** + * Parse components for the specified service from the stack definition. + * + * @param service service name + * + * @throws org.apache.ambari.server.AmbariException an exception occurred getting components from the stack definition + */ + private void parseComponents(String service) throws AmbariException{ + Collection<String> componentSet = new HashSet<String>(); + + Set<StackServiceComponentResponse> components = ambariManagementController.getStackComponents( + Collections.singleton(new StackServiceComponentRequest(name, version, service, null))); + + // stack service components + for (StackServiceComponentResponse component : components) { + String componentName = component.getComponentName(); + componentSet.add(componentName); + componentService.put(componentName, service); + String cardinality = component.getCardinality(); + if (cardinality != null) { + cardinalityRequirements.put(componentName, cardinality); + } + AutoDeployInfo autoDeploy = component.getAutoDeploy(); + if (autoDeploy != null) { + componentAutoDeployInfo.put(componentName, autoDeploy); + } + + // populate component dependencies + Collection<DependencyInfo> componentDependencies = BaseBlueprintProcessor.stackInfo.getComponentDependencies( + name, version, service, componentName); + + if (componentDependencies != null && ! componentDependencies.isEmpty()) { + dependencies.put(componentName, componentDependencies); + } + } + this.serviceComponents.put(service, componentSet); + } + + /** + * Parse configurations for the specified service from the stack definition. + * + * @param service service name + * + * @throws org.apache.ambari.server.AmbariException an exception occurred getting configurations from the stack definition + */ + private void parseConfigurations(String service) throws AmbariException { + Map<String, Map<String, ConfigProperty>> mapServiceConfig = new HashMap<String, Map<String, ConfigProperty>>(); + + serviceConfigurations.put(service, mapServiceConfig); + + Set<StackConfigurationResponse> serviceConfigs = ambariManagementController.getStackConfigurations( + Collections.singleton(new StackConfigurationRequest(name, version, service, null))); + Set<StackConfigurationResponse> stackLevelConfigs = ambariManagementController.getStackLevelConfigurations( + Collections.singleton(new StackLevelConfigurationRequest(name, version, null))); + serviceConfigs.addAll(stackLevelConfigs); + + for (StackConfigurationResponse config : serviceConfigs) { + String type = config.getType(); + //strip .xml from type + if (type.endsWith(".xml")) { + type = type.substring(0, type.length() - 4); + } + Map<String, ConfigProperty> mapTypeConfig = mapServiceConfig.get(type); + if (mapTypeConfig == null) { + mapTypeConfig = new HashMap<String, ConfigProperty>(); + mapServiceConfig.put(type, mapTypeConfig); + } + mapTypeConfig.put(config.getPropertyName(), + new ConfigProperty(config.getPropertyValue(), config.getPropertyAttributes())); + } + } + + /** + * Register conditional dependencies. + */ + //todo: This information should be specified in the stack definition. + void registerConditionalDependencies() { + Collection<DependencyInfo> nagiosDependencies = getDependenciesForComponent("NAGIOS_SERVER"); + for (DependencyInfo dependency : nagiosDependencies) { + if (dependency.getComponentName().equals("HCAT")) { + dependencyConditionalServiceMap.put(dependency, "HIVE"); + } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { + dependencyConditionalServiceMap.put(dependency, "OOZIE"); + } else if (dependency.getComponentName().equals("YARN_CLIENT")) { + dependencyConditionalServiceMap.put(dependency, "YARN"); + } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { + dependencyConditionalServiceMap.put(dependency, "TEZ"); + } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { + dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); + } + } + dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database"); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java index be5aea8..7eb28df 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java @@ -61,8 +61,8 @@ public class BaseBlueprintProcessorTest { mockSupport.replayAll(); // create stack for testing - BaseBlueprintProcessor.Stack testStack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockMgmtController) { + Stack testStack = + new Stack("HDP", "2.1", mockMgmtController) { @Override public Collection<DependencyInfo> getDependenciesForComponent(String component) { // simulate the dependencies in a given stack by overriding this method @@ -123,8 +123,8 @@ public class BaseBlueprintProcessorTest { mockSupport.replayAll(); // create stack for testing - BaseBlueprintProcessor.Stack testStack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockMgmtController) { + Stack testStack = + new Stack("HDP", "2.1", mockMgmtController) { @Override public Collection<DependencyInfo> getDependenciesForComponent(String component) { // simulate the dependencies in a given stack by overriding this method @@ -182,8 +182,8 @@ public class BaseBlueprintProcessorTest { mockSupport.replayAll(); // create stack for testing - BaseBlueprintProcessor.Stack testStack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockMgmtController) { + Stack testStack = + new Stack("HDP", "2.1", mockMgmtController) { @Override public Collection<DependencyInfo> getDependenciesForComponent(String component) { // simulate the dependencies in a given stack by overriding this method @@ -241,8 +241,8 @@ public class BaseBlueprintProcessorTest { mockSupport.replayAll(); // create stack for testing - BaseBlueprintProcessor.Stack testStack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockMgmtController) { + Stack testStack = + new Stack("HDP", "2.1", mockMgmtController) { @Override public Collection<DependencyInfo> getDependenciesForComponent(String component) { // simulate the dependencies in a given stack by overriding this method @@ -300,8 +300,8 @@ public class BaseBlueprintProcessorTest { mockSupport.replayAll(); // create stack for testing - BaseBlueprintProcessor.Stack testStack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockMgmtController) { + Stack testStack = + new Stack("HDP", "2.1", mockMgmtController) { @Override public Collection<DependencyInfo> getDependenciesForComponent(String component) { // simulate the dependencies in a given stack by overriding this method @@ -359,8 +359,8 @@ public class BaseBlueprintProcessorTest { mockSupport.replayAll(); // create stack for testing - BaseBlueprintProcessor.Stack testStack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockMgmtController) { + Stack testStack = + new Stack("HDP", "2.1", mockMgmtController) { @Override public Collection<DependencyInfo> getDependenciesForComponent(String component) { // simulate the dependencies in a given stack by overriding this method http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 260d043..04c75a1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.controller.internal; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.StackServiceResponse; import org.easymock.EasyMockSupport; import org.junit.Test; @@ -33,7 +35,9 @@ import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertNotNull; import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.fail; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.isA; /** * BlueprintConfigurationProcessor unit tests. @@ -327,12 +331,183 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("testhost", updatedVal); } @Test + public void testDoUpdateForClusterCreate_SingleHostProperty__MissingComponent() throws Exception { + EasyMockSupport mockSupport = new EasyMockSupport(); + + + AmbariManagementController mockMgmtController = + mockSupport.createMock(AmbariManagementController.class); + + expect(mockMgmtController.getStackServices(isA(Set.class))).andReturn(Collections.<StackServiceResponse>emptySet()); + + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("yarn.resourcemanager.hostname", "localhost"); + typeProps.put("yarn.timeline-service.address", "localhost"); + properties.put("yarn-site", typeProps); + + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("NAMENODE"); + hgComponents.add("SECONDARY_NAMENODE"); + hgComponents.add("RESOURCEMANAGER"); + HostGroup group1 = new TestHostGroup("group1", Collections.singleton("testhost"), hgComponents); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("DATANODE"); + hgComponents2.add("HDFS_CLIENT"); + HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2); + + Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>(); + hostGroups.put(group1.getName(), group1); + hostGroups.put(group2.getName(), group2); + + mockSupport.replayAll(); + + Stack testStackDefinition = new Stack("HDP", "2.1", mockMgmtController) { + @Override + public Cardinality getCardinality(String component) { + // simulate a stack that required the APP_TIMELINE_SERVER + if (component.equals("APP_TIMELINE_SERVER")) { + return new Cardinality("1"); + } + + return null; + } + }; + + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); + + try { + updater.doUpdateForClusterCreate(hostGroups, testStackDefinition); + fail("IllegalArgumentException should have been thrown"); + } catch (IllegalArgumentException illegalArgumentException) { + // expected exception + } + + mockSupport.verifyAll(); + } + + @Test + public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleMatchingHostGroupsError() throws Exception { + EasyMockSupport mockSupport = new EasyMockSupport(); + + + AmbariManagementController mockMgmtController = + mockSupport.createMock(AmbariManagementController.class); + + expect(mockMgmtController.getStackServices(isA(Set.class))).andReturn(Collections.<StackServiceResponse>emptySet()); + + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("yarn.resourcemanager.hostname", "localhost"); + typeProps.put("yarn.timeline-service.address", "localhost"); + properties.put("yarn-site", typeProps); + + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("NAMENODE"); + hgComponents.add("SECONDARY_NAMENODE"); + hgComponents.add("RESOURCEMANAGER"); + hgComponents.add("APP_TIMELINE_SERVER"); + HostGroup group1 = new TestHostGroup("group1", Collections.singleton("testhost"), hgComponents); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("DATANODE"); + hgComponents2.add("HDFS_CLIENT"); + hgComponents2.add("APP_TIMELINE_SERVER"); + HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2); + + Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>(); + hostGroups.put(group1.getName(), group1); + hostGroups.put(group2.getName(), group2); + + mockSupport.replayAll(); + + Stack testStackDefinition = new Stack("HDP", "2.1", mockMgmtController) { + @Override + public Cardinality getCardinality(String component) { + // simulate a stack that required the APP_TIMELINE_SERVER + if (component.equals("APP_TIMELINE_SERVER")) { + return new Cardinality("0-1"); + } + + return null; + } + }; + + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); + + try { + updater.doUpdateForClusterCreate(hostGroups, testStackDefinition); + fail("IllegalArgumentException should have been thrown"); + } catch (IllegalArgumentException illegalArgumentException) { + // expected exception + } + + mockSupport.verifyAll(); + } + + @Test + public void testDoUpdateForClusterCreate_SingleHostProperty__MissingOptionalComponent() throws Exception { + final String expectedHostName = "localhost"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + AmbariManagementController mockMgmtController = + mockSupport.createMock(AmbariManagementController.class); + + expect(mockMgmtController.getStackServices(isA(Set.class))).andReturn(Collections.<StackServiceResponse>emptySet()); + + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> typeProps = new HashMap<String, String>(); + typeProps.put("yarn.timeline-service.address", expectedHostName); + properties.put("yarn-site", typeProps); + + Collection<String> hgComponents = new HashSet<String>(); + hgComponents.add("NAMENODE"); + hgComponents.add("SECONDARY_NAMENODE"); + hgComponents.add("RESOURCEMANAGER"); + HostGroup group1 = new TestHostGroup("group1", Collections.singleton("testhost"), hgComponents); + + Collection<String> hgComponents2 = new HashSet<String>(); + hgComponents2.add("DATANODE"); + hgComponents2.add("HDFS_CLIENT"); + HostGroup group2 = new TestHostGroup("group2", Collections.singleton("testhost2"), hgComponents2); + + Map<String, HostGroup> hostGroups = new HashMap<String, HostGroup>(); + hostGroups.put(group1.getName(), group1); + hostGroups.put(group2.getName(), group2); + + mockSupport.replayAll(); + + Stack testStackDefinition = new Stack("HDP", "2.1", mockMgmtController) { + @Override + public Cardinality getCardinality(String component) { + // simulate a stack that supports 0 or 1 instances of the APP_TIMELINE_SERVER + if (component.equals("APP_TIMELINE_SERVER")) { + return new Cardinality("0-1"); + } + + return null; + } + }; + + BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); + + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, testStackDefinition); + String updatedVal = updatedProperties.get("yarn-site").get("yarn.timeline-service.address"); + assertEquals("Timeline Server config property should not have been updated", + expectedHostName, updatedVal); + + mockSupport.verifyAll(); + } + + @Test public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue__WithPort() { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); Map<String, String> typeProps = new HashMap<String, String>(); @@ -355,7 +530,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("core-site").get("fs.defaultFS"); assertEquals("testhost:5050", updatedVal); } @@ -397,7 +572,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group3.getName(), group3); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hbase-site").get("hbase.zookeeper.quorum"); String[] hosts = updatedVal.split(","); @@ -451,7 +626,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group3.getName(), group3); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("webhcat-site").get("templeton.zookeeper.hosts"); String[] hosts = updatedVal.split(","); @@ -505,7 +680,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group3.getName(), group3); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("storm-site").get("storm.zookeeper.servers"); assertTrue(updatedVal.startsWith("[")); assertTrue(updatedVal.endsWith("]")); @@ -550,7 +725,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hbase-env").get("hbase_master_heapsize"); assertEquals("512m", updatedVal); } @@ -578,7 +753,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hbase-env").get("hbase_master_heapsize"); assertEquals("512m", updatedVal); } @@ -606,7 +781,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("yarn-site").get("yarn.resourcemanager.hostname"); assertEquals("testhost", updatedVal); } @@ -634,7 +809,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("core-site").get("fs.defaultFS"); assertEquals("testhost:5050", updatedVal); } @@ -676,7 +851,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group3.getName(), group3); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hbase-site").get("hbase.zookeeper.quorum"); String[] hosts = updatedVal.split(","); @@ -730,7 +905,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group3.getName(), group3); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("webhcat-site").get("templeton.zookeeper.hosts"); String[] hosts = updatedVal.split(","); @@ -784,7 +959,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group3.getName(), group3); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("storm-site").get("storm.zookeeper.servers"); assertTrue(updatedVal.startsWith("[")); assertTrue(updatedVal.endsWith("]")); @@ -833,7 +1008,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hive-site").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql://testhost/hive?createDatabaseIfNotExist=true", updatedVal); } @@ -865,7 +1040,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hive-site").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql://testhost/hive?createDatabaseIfNotExist=true", updatedVal); } @@ -894,7 +1069,7 @@ public class BlueprintConfigurationProcessorTest { hostGroups.put(group2.getName(), group2); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(properties); - Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups); + Map<String, Map<String, String>> updatedProperties = updater.doUpdateForClusterCreate(hostGroups, null); String updatedVal = updatedProperties.get("hive-env").get("javax.jdo.option.ConnectionURL"); assertEquals("jdbc:mysql://myHost.com/hive?createDatabaseIfNotExist=true", updatedVal); } @@ -981,7 +1156,7 @@ public class BlueprintConfigurationProcessorTest { mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne); // call top-level export method - configProcessor.doUpdateForClusterCreate(mapOfHostGroups); + configProcessor.doUpdateForClusterCreate(mapOfHostGroups, null); assertEquals("Falcon Broker URL property not properly exported", expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url")); @@ -1031,7 +1206,7 @@ public class BlueprintConfigurationProcessorTest { mapOfHostGroups.put(expectedHostGroupName, mockHostGroupOne); // call top-level export method - configProcessor.doUpdateForClusterCreate(mapOfHostGroups); + configProcessor.doUpdateForClusterCreate(mapOfHostGroups, null); assertEquals("Falcon Broker URL property not properly exported", expectedHostName + ":" + expectedPortNum, falconStartupProperties.get("*.broker.url")); @@ -1091,7 +1266,7 @@ public class BlueprintConfigurationProcessorTest { Map<String, HostGroup> mapOfHostGroups = new HashMap<String, HostGroup>(); mapOfHostGroups.put(expectedHostGroupName,mockHostGroupOne); - configProcessor.doUpdateForClusterCreate(mapOfHostGroups); + configProcessor.doUpdateForClusterCreate(mapOfHostGroups, null); // verify that the expected hostname was substitued for the host group name in the config assertEquals("HTTPS address HA property not properly exported", http://git-wip-us.apache.org/repos/asf/ambari/blob/c19b7896/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java index c1185ae..319d3a9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterResourceProviderTest.java @@ -3070,8 +3070,8 @@ public class ClusterResourceProviderTest { ClusterResourceProvider.init(null, mockAmbariMetaInfo, null); - BaseBlueprintProcessor.Stack stack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockManagementController); + Stack stack = + new Stack("HDP", "2.1", mockManagementController); ClusterResourceProvider clusterResourceProvider = new TestClusterResourceProvider(mockMgmtController, mockServiceProvider, @@ -3152,8 +3152,8 @@ public class ClusterResourceProviderTest { ClusterResourceProvider.init(null, mockAmbariMetaInfo, null); - BaseBlueprintProcessor.Stack stack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockManagementController); + Stack stack = + new Stack("HDP", "2.1", mockManagementController); ClusterResourceProvider clusterResourceProvider = new TestClusterResourceProvider(mockMgmtController, mockServiceProvider, @@ -3233,8 +3233,8 @@ public class ClusterResourceProviderTest { ClusterResourceProvider.init(null, mockAmbariMetaInfo, null); - BaseBlueprintProcessor.Stack stack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockManagementController); + Stack stack = + new Stack("HDP", "2.1", mockManagementController); ClusterResourceProvider clusterResourceProvider = new TestClusterResourceProvider(mockMgmtController, mockServiceProvider, @@ -3310,8 +3310,8 @@ public class ClusterResourceProviderTest { ClusterResourceProvider.init(null, mockAmbariMetaInfo, null); - BaseBlueprintProcessor.Stack stack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockManagementController); + Stack stack = + new Stack("HDP", "2.1", mockManagementController); ClusterResourceProvider clusterResourceProvider = new TestClusterResourceProvider(mockMgmtController, mockServiceProvider, @@ -3386,8 +3386,8 @@ public class ClusterResourceProviderTest { ClusterResourceProvider.init(null, mockAmbariMetaInfo, null); - BaseBlueprintProcessor.Stack stack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockManagementController); + Stack stack = + new Stack("HDP", "2.1", mockManagementController); ClusterResourceProvider clusterResourceProvider = new TestClusterResourceProvider(mockMgmtController, mockServiceProvider, @@ -3462,8 +3462,8 @@ public class ClusterResourceProviderTest { ClusterResourceProvider.init(null, mockAmbariMetaInfo, null); - BaseBlueprintProcessor.Stack stack = - new BaseBlueprintProcessor.Stack("HDP", "2.1", mockManagementController); + Stack stack = + new Stack("HDP", "2.1", mockManagementController); ClusterResourceProvider clusterResourceProvider = new TestClusterResourceProvider(mockMgmtController, mockServiceProvider,
