Repository: ambari Updated Branches: refs/heads/trunk 987373628 -> 11db1a425
AMBARI-13582. Allow use passwords references in custom actions. (vbrodetskyi via yusaku) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/11db1a42 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/11db1a42 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/11db1a42 Branch: refs/heads/trunk Commit: 11db1a4258b78f22e5bd3892239246a9989ef4f6 Parents: 9873736 Author: Yusaku Sako <yus...@hortonworks.com> Authored: Wed Oct 28 11:42:54 2015 -0700 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Wed Oct 28 11:42:54 2015 -0700 ---------------------------------------------------------------------- .../controller/AmbariActionExecutionHelper.java | 4 + .../AmbariManagementControllerImpl.java | 27 ++++--- .../ambari/server/controller/AmbariServer.java | 2 +- .../controller/ConfigurationResponse.java | 13 +--- .../internal/BlueprintResourceProvider.java | 49 +++++++++--- .../apache/ambari/server/state/StackInfo.java | 19 +++++ .../server/state/cluster/ClusterImpl.java | 14 +--- .../server/topology/BlueprintValidatorImpl.java | 27 +++++++ .../ambari/server/utils/SecretReference.java | 78 +++++++++++++++----- .../AmbariManagementControllerTest.java | 24 ++++-- .../internal/BlueprintResourceProviderTest.java | 47 ++++++++++-- .../server/topology/BlueprintImplTest.java | 63 +++++++++++++++- .../services/YARN/configuration/yarn-site.xml | 6 ++ 13 files changed, 291 insertions(+), 82 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index d834731..215aca8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -46,6 +46,7 @@ import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent; +import org.apache.ambari.server.utils.SecretReference; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -406,6 +407,9 @@ public class AmbariActionExecutionHelper { } roleParams.putAll(actionContext.getParameters()); + + SecretReference.replaceReferencesWithPasswords(roleParams, cluster); + if (componentInfo != null) { roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 3a04a90..8099c64 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -729,9 +729,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String passwordPropertyValue = requestProperties.get(passwordProperty); if (!SecretReference.isSecret(passwordPropertyValue)) continue; - SecretReference ref = new SecretReference(passwordPropertyValue, passwordProperty, cluster); - if (!ref.getClusterName().equals(request.getClusterName())) - throw new AmbariException("Can not reference to different cluster in SECRET"); + SecretReference ref = new SecretReference(passwordPropertyValue, cluster); String refValue = ref.getValue(); requestProperties.put(passwordProperty, refValue); } @@ -932,9 +930,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (LOG.isDebugEnabled()) { LOG.debug("Received a getClusters request" - + ", clusterName=" + request.getClusterName() - + ", clusterId=" + request.getClusterId() - + ", stackInfo=" + request.getStackVersion()); + + ", clusterName=" + request.getClusterName() + + ", clusterId=" + request.getClusterId() + + ", stackInfo=" + request.getStackVersion()); } Cluster singleCluster = null; @@ -1401,7 +1399,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (propertiesTypes.containsKey(PropertyType.PASSWORD) && propertiesTypes.get(PropertyType.PASSWORD).contains(propertyName)) { if (SecretReference.isSecret(propertyValue)) { - SecretReference ref = new SecretReference(propertyValue, propertyName, cluster); + SecretReference ref = new SecretReference(propertyValue, cluster); requestConfigProperties.put(propertyName, ref.getValue()); } } @@ -3399,30 +3397,31 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle ExecuteCommandJson jsons = customCommandExecutionHelper.getCommandJson(actionExecContext, cluster, stackId); String commandParamsForStage = jsons.getCommandParamsForStage(); + Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, new TypeToken<Map<String, String>>() + {}.getType()); // Ensure that the specified requestContext (if any) is set as the request context if (!requestContext.isEmpty()) { requestStageContainer.setRequestContext(requestContext); } + // replace password references in requestProperties + SecretReference.replaceReferencesWithPasswords(commandParamsStage, cluster); + // If the request is to perform the Kerberos service check, set up the stages to // ensure that the (cluster-level) smoke user principal and keytab is available on all hosts boolean kerberosServiceCheck = Role.KERBEROS_SERVICE_CHECK.name().equals(actionRequest.getCommandName()); if (kerberosServiceCheck) { // Parse the command parameters into a map so that additional values may be added to it - Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, - new TypeToken<Map<String, String>>() { - }.getType()); try { requestStageContainer = kerberosHelper.createTestIdentity(cluster, commandParamsStage, requestStageContainer); } catch (KerberosOperationException e) { throw new IllegalArgumentException(e.getMessage(), e); } - - // Recreate commandParamsForStage with the added values - commandParamsForStage = gson.toJson(commandParamsStage); } + commandParamsForStage = gson.toJson(commandParamsStage); + Stage stage = createNewStage(requestStageContainer.getLastStageId(), cluster, requestId, requestContext, jsons.getClusterHostInfo(), commandParamsForStage, jsons.getHostParamsForStage()); @@ -3453,7 +3452,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (kerberosServiceCheck) { // Parse the command parameters into a map so that existing values may be accessed and // additional values may be added to it. - Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, + commandParamsStage = gson.fromJson(commandParamsForStage, new TypeToken<Map<String, String>>() { }.getType()); http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index 12373f0..484cb39 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -686,7 +686,7 @@ public class AmbariServer { StackDefinedPropertyProvider.init(injector); AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class)); BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class), - injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class)); + injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class), ambariMetaInfo); StackDependencyResourceProvider.init(ambariMetaInfo); ClusterResourceProvider.init(injector.getInstance(TopologyManager.class), injector.getInstance(TopologyRequestFactoryImpl.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java index 3ed9306..eef3474 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java @@ -79,7 +79,7 @@ public class ConfigurationResponse { this.configs = configs; this.configAttributes = configAttributes; this.propertiesTypes = propertiesTypes; - stubPasswords(); + SecretReference.replacePasswordsWithReferences(propertiesTypes, configs, type, version); } /** @@ -215,15 +215,4 @@ public class ConfigurationResponse { public void setPropertiesTypes(Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes) { this.propertiesTypes = propertiesTypes; } - - private void stubPasswords(){ - if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD)) { - for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD)) { - if(configs.containsKey(pwdPropertyName)){ - String stub = SecretReference.generateStub(clusterName, type, version); - configs.put(pwdPropertyName, stub); - } - } - } - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 6cb6a74..5994094 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -32,6 +32,7 @@ import java.util.Set; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.DuplicateResourceException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.NoSuchResourceException; @@ -51,11 +52,13 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.stack.NoSuchStackException; +import org.apache.ambari.server.state.*; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; import org.apache.ambari.server.topology.InvalidTopologyException; import com.google.gson.Gson; +import org.apache.ambari.server.utils.SecretReference; /** @@ -116,6 +119,10 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide */ private static Gson jsonSerializer; + /** + * Used to get stack metainfo. + */ + private static AmbariMetaInfo ambariMetaInfo; // ----- Constructors ---------------------------------------------------- @@ -140,10 +147,11 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * @param dao blueprint data access object * @param gson json serializer */ - public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson) { + public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson, AmbariMetaInfo metaInfo) { blueprintFactory = factory; blueprintDAO = dao; jsonSerializer = gson; + ambariMetaInfo = metaInfo; } // ----- ResourceProvider ------------------------------------------------ @@ -175,7 +183,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide //todo: continue to use dao/entity directly or use blueprint factory? public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, - NoSuchResourceException, NoSuchParentResourceException { + NoSuchResourceException, NoSuchParentResourceException { List<BlueprintEntity> results = null; boolean applyPredicate = false; @@ -242,8 +250,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide modifyResources(new Command<Void>() { @Override public Void invoke() throws AmbariException { - blueprintDAO.removeByName(blueprintName); - return null; + blueprintDAO.removeByName(blueprintName); + return null; } }); } @@ -263,7 +271,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * * @return a new resource instance for the given blueprint entity */ - protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) { + protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) throws NoSuchResourceException { StackEntity stackEntity = entity.getStack(); Resource resource = new ResourceImpl(Resource.Type.Blueprint); setResourceProperty(resource, BLUEPRINT_NAME_PROPERTY_ID, entity.getBlueprintName(), requestedIds); @@ -304,16 +312,39 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * @return list of configuration property maps */ List<Map<String, Map<String, Object>>> populateConfigurationList( - Collection<? extends BlueprintConfiguration> configurations) { + Collection<? extends BlueprintConfiguration> configurations) throws NoSuchResourceException { List<Map<String, Map<String, Object>>> listConfigurations = new ArrayList<Map<String, Map<String, Object>>>(); for (BlueprintConfiguration config : configurations) { Map<String, Map<String, Object>> mapConfigurations = new HashMap<String, Map<String, Object>>(); Map<String, Object> configTypeDefinition = new HashMap<String, Object>(); String type = config.getType(); - Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson( - config.getConfigData(), Map.class); - configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); + + if(config instanceof BlueprintConfigEntity) { + Map<String, String> properties = jsonSerializer.<Map<String, String>>fromJson( + config.getConfigData(), Map.class); + + StackEntity stack = ((BlueprintConfigEntity)config).getBlueprintEntity().getStack(); + StackInfo metaInfoStack; + + try { + metaInfoStack = ambariMetaInfo.getStack(stack.getStackName(), stack.getStackVersion()); + } catch (AmbariException e) { + throw new NoSuchResourceException(e.getMessage()); + } + + Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes = + metaInfoStack.getConfigPropertiesTypes(type); + + SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1l); + + configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); + } else { + Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson( + config.getConfigData(), Map.class); + configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); + } + Map<String, Map<String, String>> attributes = jsonSerializer.<Map<String, Map<String, String>>>fromJson( config.getConfigAttributes(), Map.class); if (attributes != null && !attributes.isEmpty()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index e3ac3e0..2b9cd83 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -436,4 +436,23 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } return result; } + + public Map<PropertyInfo.PropertyType, Set<String>> getConfigPropertiesTypes(String configType) { + Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes = new HashMap<>(); + Collection<ServiceInfo> services = getServices(); + for (ServiceInfo serviceInfo : services) { + for (PropertyInfo propertyInfo : serviceInfo.getProperties()) { + if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty()) { + Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes(); + for (PropertyInfo.PropertyType propertyType : types) { + if (!propertiesTypes.containsKey(propertyType)) + propertiesTypes.put(propertyType, new HashSet<String>()); + propertiesTypes.get(propertyType).add(propertyInfo.getName()); + } + } + } + } + return propertiesTypes; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 279b31f..c216110 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -2942,19 +2942,7 @@ public class ClusterImpl implements Cluster { try { StackId stackId = this.getCurrentStackVersion(); StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); - Collection<ServiceInfo> services = stackInfo.getServices(); - for (ServiceInfo serviceInfo : services) { - for (PropertyInfo propertyInfo : serviceInfo.getProperties()) { - if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty()) { - Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes(); - for (PropertyInfo.PropertyType propertyType : types) { - if (!propertiesTypes.containsKey(propertyType)) - propertiesTypes.put(propertyType, new HashSet<String>()); - propertiesTypes.get(propertyType).add(propertyInfo.getName()); - } - } - } - } + propertiesTypes = stackInfo.getConfigPropertiesTypes(configType); } catch (Exception e) { } http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index 9e8f163..1c293ee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -21,6 +21,7 @@ package org.apache.ambari.server.topology; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyInfo; +import org.apache.ambari.server.utils.SecretReference; import org.apache.ambari.server.utils.VersionUtils; import java.util.Collection; @@ -83,6 +84,32 @@ public class BlueprintValidatorImpl implements BlueprintValidator { // we don't want to include default stack properties so we can't just use hostGroup full properties Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); + // we need to have real passwords, not references + if(clusterConfigurations != null) { + StringBuilder errorMessage = new StringBuilder(); + boolean containsSecretReferences = false; + for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { + String configType = configEntry.getKey(); + if (configEntry.getValue() != null) { + for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { + String propertyName = propertyEntry.getKey(); + String propertyValue = propertyEntry.getValue(); + if (propertyValue != null) { + if (SecretReference.isSecret(propertyValue)) { + errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); + containsSecretReferences = true; + } + } + } + } + } + if(containsSecretReferences) { + throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + + "replace following properties with real passwords:\n"+errorMessage.toString()); + } + } + + for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java index 2b1aeae..d801975 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java @@ -21,37 +21,36 @@ package org.apache.ambari.server.utils; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.PropertyInfo; import java.util.Map; +import java.util.Set; public class SecretReference { - private String clusterName; + private static final String secretPrefix = "SECRET"; private String configType; private Long version; private String value; - private String reference; - public SecretReference(String reference, String propertyName, Cluster cluster) throws AmbariException{ + public SecretReference(String reference, Cluster cluster) throws AmbariException{ String[] values = reference.split(":"); - clusterName = values[1]; - configType = values[2]; - version = Long.valueOf(values[3]); + + configType = values[1]; + version = Long.valueOf(values[2]); + + String propertyName = values[3]; + String clusterName = cluster.getClusterName(); Config refConfig = cluster.getConfigByVersion(configType, version); if(refConfig == null) - throw new AmbariException(String.format("Cluster: %s does not contain ConfigType: %s ConfigVersion: %s", - cluster.getClusterName(), configType, version)); + throw new AmbariException(String.format("Error when parsing secret reference. Cluster: %s does not contain ConfigType: %s ConfigVersion: %s", + clusterName, configType, version)); Map<String, String> refProperties = refConfig.getProperties(); if(!refProperties.containsKey(propertyName)) - throw new AmbariException(String.format("Cluster: %s ConfigType: %s ConfigVersion: %s does not contain property '%s'", - cluster.getClusterName(), configType, version, propertyName)); - this.value = refProperties.get(propertyName); - - this.reference = reference; - } + throw new AmbariException(String.format("Error when parsing secret reference. Cluster: %s ConfigType: %s ConfigVersion: %s does not contain property '%s'", + clusterName, configType, version, propertyName)); - public String getClusterName() { - return clusterName; + this.value = refProperties.get(propertyName); } public void setConfigType(String configType) { @@ -68,10 +67,51 @@ public class SecretReference { public static boolean isSecret(String value) { String[] values = value.split(":"); - return values.length == 4 && values[0].equals("SECRET"); + return values.length == 4 && values[0].equals(secretPrefix); + } + + public static String generateStub(String configType, Long configVersion, String propertyName) { + return secretPrefix + ":" + configType + ":" + configVersion.toString() + ":" + propertyName; + } + + /** + * Replace secret references with appropriate real passwords. + * @param targetMap map in which replacement will be performed + * @param cluster current cluster + * @throws AmbariException + */ + public static void replaceReferencesWithPasswords(Map<String, String> targetMap, Cluster cluster) + throws AmbariException { + if(cluster != null) { + for (Map.Entry<String, String> propertyValueEntry : targetMap.entrySet()) { + String key = propertyValueEntry.getKey(); + String value = propertyValueEntry.getValue(); + if (value != null && SecretReference.isSecret(value)) { + SecretReference ref = new SecretReference(value, cluster); + targetMap.put(key, ref.getValue()); + } + } + } } - public static String generateStub(String clusterName, String configType, Long configVersion) { - return "SECRET:" + clusterName + ":" + configType + ":" + configVersion.toString(); + /** + * Replace real passwords with secret references + * @param propertiesTypes map with properties types + * @param propertiesMap map with properties in which replacement will be performed + * @param configType configuration type + * @param configVersion configuration version + */ + public static void replacePasswordsWithReferences(Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes, + Map<String, String> propertiesMap, + String configType, + Long configVersion){ + if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD)) { + for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD)) { + if(propertiesMap.containsKey(pwdPropertyName)){ + String stub = SecretReference.generateStub(configType, configVersion, pwdPropertyName); + propertiesMap.put(pwdPropertyName, stub); + } + } + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index af8c5e2..5de7737 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -4123,7 +4123,7 @@ public class AmbariManagementControllerTest { Config config1 = cf.createNew(cluster, "global", new HashMap<String, String>() {{ put("key1", "value1"); - }}, new HashMap<String, Map<String,String>>()); + }}, new HashMap<String, Map<String, String>>()); config1.setTag("version1"); Config config2 = cf.createNew(cluster, "core-site", @@ -4132,8 +4132,15 @@ public class AmbariManagementControllerTest { }}, new HashMap<String, Map<String,String>>()); config2.setTag("version1"); + Config config3 = cf.createNew(cluster, "yarn-site", + new HashMap<String, String>() {{ + put("test.password", "supersecret"); + }}, new HashMap<String, Map<String,String>>()); + config3.setTag("version1"); + cluster.addConfig(config1); cluster.addConfig(config2); + cluster.addConfig(config3); Service hdfs = cluster.addService("HDFS"); hdfs.persist(); @@ -4161,6 +4168,7 @@ public class AmbariManagementControllerTest { Map<String, String> params = new HashMap<String, String>() {{ put("test", "test"); + put("pwd", "SECRET:yarn-site:1:test.password"); }}; Map<String, String> requestProperties = new HashMap<String, String>(); @@ -4193,6 +4201,8 @@ public class AmbariManagementControllerTest { Map<String, String> commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); Assert.assertTrue(commandParametersStage.containsKey("test")); + Assert.assertTrue(commandParametersStage.containsKey("pwd")); + Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret"); Assert.assertEquals("HDFS", cmd.getServiceName()); Assert.assertEquals("DATANODE", cmd.getComponentName()); Assert.assertNotNull(hostParametersStage.get("jdk_location")); @@ -4233,6 +4243,8 @@ public class AmbariManagementControllerTest { commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); Assert.assertTrue(commandParametersStage.containsKey("test")); + Assert.assertTrue(commandParametersStage.containsKey("pwd")); + Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret"); Assert.assertEquals("HDFS", cmd.getServiceName()); Assert.assertEquals("DATANODE", cmd.getComponentName()); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); @@ -10589,7 +10601,7 @@ public class AmbariManagementControllerTest { "hdfs-site", "version2", new HashMap<String, String>(){{ - put("test.password", "SECRET:c1:hdfs-site:1"); + put("test.password", "SECRET:hdfs-site:1:test.password"); put("new", "new");//need this to mark config as "changed" }}, new HashMap<String, Map<String, String>>() @@ -10614,7 +10626,7 @@ public class AmbariManagementControllerTest { "hdfs-site", "version3", new HashMap<String, String>(){{ - put("test.password", "SECRET:c1:hdfs-site:666"); + put("test.password", "SECRET:hdfs-site:666:test.password"); }}, new HashMap<String, Map<String, String>>() ); @@ -10642,7 +10654,7 @@ public class AmbariManagementControllerTest { "hdfs-site", "version5", new HashMap<String, String>(){{ - put("test.password", "SECRET:c1:hdfs-site:4"); + put("test.password", "SECRET:hdfs-site:4:test.password"); put("new", "new"); }}, new HashMap<String, Map<String, String>>() @@ -10653,7 +10665,7 @@ public class AmbariManagementControllerTest { controller.updateClusters(Collections.singleton(crReq), null); fail("Request need to be failed with wrong secret reference"); } catch (AmbariException e) { - assertEquals("Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain property 'test.password'", + assertEquals("Error when parsing secret reference. Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain property 'test.password'", e.getMessage()); } cl.getAllConfigs(); @@ -10676,7 +10688,7 @@ public class AmbariManagementControllerTest { add(configRequest); }}); for(ConfigurationResponse resp : requestedConfigs) { - String secretName = "SECRET:foo1:hdfs-site:"+resp.getVersion().toString(); + String secretName = "SECRET:hdfs-site:"+resp.getVersion().toString()+":test.password"; if(resp.getConfigs().containsKey("test.password")) { assertEquals(resp.getConfigs().get("test.password"), secretName); } http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java index 5bfdebb..8ef4bbb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java @@ -44,6 +44,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategy; @@ -69,6 +70,8 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.state.*; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; @@ -104,7 +107,7 @@ public class BlueprintResourceProviderTest { @BeforeClass public static void initClass() { - BlueprintResourceProvider.init(blueprintFactory, dao, gson); + BlueprintResourceProvider.init(blueprintFactory, dao, gson, metaInfo); StackEntity stackEntity = new StackEntity(); stackEntity.setStackName("test-stack-name"); @@ -381,7 +384,12 @@ public class BlueprintResourceProviderTest { @Test public void testGetResourcesNoPredicate_withConfiguration() throws SystemException, UnsupportedPropertyException, - NoSuchParentResourceException, NoSuchResourceException { + NoSuchParentResourceException, NoSuchResourceException, AmbariException { + + StackInfo info = createMock(StackInfo.class); + expect(info.getConfigPropertiesTypes("core-site")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes(); + expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes(); + replay(info, metaInfo); Request request = createNiceMock(Request.class); Set<Map<String, Object>> testProperties = getBlueprintTestProperties(); @@ -951,20 +959,44 @@ public class BlueprintResourceProviderTest { @Test public void testPopulateConfigurationList() throws Exception { + StackEntity stackEntity = new StackEntity(); + stackEntity.setStackName("test-stack-name"); + stackEntity.setStackVersion("test-stack-version"); + BlueprintEntity entity = createMock(BlueprintEntity.class); + expect(entity.getStack()).andReturn(stackEntity).anyTimes(); + + HashMap<PropertyInfo.PropertyType, Set<String>> pwdProperties = new HashMap<PropertyInfo.PropertyType, Set<String>>() {{ + put(PropertyInfo.PropertyType.PASSWORD, new HashSet<String>(){{ + add("test.password"); + }}); + }}; + + StackInfo info = createMock(StackInfo.class); + expect(info.getConfigPropertiesTypes("type1")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes(); + expect(info.getConfigPropertiesTypes("type2")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes(); + expect(info.getConfigPropertiesTypes("type3")).andReturn(pwdProperties).anyTimes(); + expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes(); + + replay(info, metaInfo, entity); + + // attributes is null - BlueprintConfiguration config1 = new BlueprintConfigEntity(); + BlueprintConfigEntity config1 = new BlueprintConfigEntity(); config1.setType("type1"); config1.setConfigData("{\"key1\":\"value1\"}"); + config1.setBlueprintEntity(entity); // attributes is empty - BlueprintConfiguration config2 = new BlueprintConfigEntity(); + BlueprintConfigEntity config2 = new BlueprintConfigEntity(); config2.setType("type2"); config2.setConfigData("{\"key2\":\"value2\"}"); config2.setConfigAttributes("{}"); + config2.setBlueprintEntity(entity); // attributes is provided - BlueprintConfiguration config3 = new BlueprintConfigEntity(); + BlueprintConfigEntity config3 = new BlueprintConfigEntity(); config3.setType("type3"); - config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\"}"); + config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\",\"test.password\":\"pwdValue\"}"); config3.setConfigAttributes("{\"final\":{\"key3\":\"attrValue1\",\"key4\":\"attrValue2\"}}"); + config3.setBlueprintEntity(entity); List<Map<String, Map<String, Object>>> configs = provider.populateConfigurationList(Arrays.asList(config1, config2, config3)); @@ -1010,9 +1042,10 @@ public class BlueprintResourceProviderTest { Map<String, String> confProperties3 = (Map<String, String>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_PROPERTY_ID); assertNotNull(confProperties3); - assertEquals(2, confProperties3.size()); + assertEquals(3, confProperties3.size()); assertEquals("value3", confProperties3.get("key3")); assertEquals("value4", confProperties3.get("key4")); + assertEquals("SECRET:type3:-1:test.password", confProperties3.get("test.password")); assertTrue(typeConfig3.containsKey(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID)); Map<String, Map<String, String>> confAttributes3 = (Map<String, Map<String, String>>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID); http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index 9d4163a..de740f4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -183,7 +183,6 @@ public class BlueprintImplTest { properties.put("hdfs-site", hdfsProps); hdfsProps.put("foo", "val"); hdfsProps.put("bar", "val"); - Map<String, String> category1Props = new HashMap<String, String>(); properties.put("category1", category1Props); category1Props.put("prop1", "val"); @@ -282,6 +281,68 @@ public class BlueprintImplTest { verify(stack, group1, group2); } + @Test + public void testValidateConfigurations__secretReference(){ + Stack stack = createNiceMock(Stack.class); + + HostGroup group1 = createNiceMock(HostGroup.class); + HostGroup group2 = createNiceMock(HostGroup.class); + Collection<HostGroup> hostGroups = new HashSet<HostGroup>(); + hostGroups.add(group1); + hostGroups.add(group2); + + Set<String> group1Components = new HashSet<String>(); + group1Components.add("c1"); + group1Components.add("c2"); + + Set<String> group2Components = new HashSet<String>(); + group2Components.add("c1"); + group2Components.add("c3"); + + Map<String, Map<String, String>> group2Props = new HashMap<String, Map<String, String>>(); + Map<String, String> group2Category2Props = new HashMap<String, String>(); + group2Props.put("category2", group2Category2Props); + group2Category2Props.put("prop2", "val"); + + Collection<Stack.ConfigProperty> requiredHDFSProperties = new HashSet<Stack.ConfigProperty>(); + requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "foo", null)); + requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "bar", null)); + requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "some_password", null)); + + requiredHDFSProperties.add(new Stack.ConfigProperty("category1", "prop1", null)); + + Collection<Stack.ConfigProperty> requiredService2Properties = new HashSet<Stack.ConfigProperty>(); + requiredService2Properties.add(new Stack.ConfigProperty("category2", "prop2", null)); + + + // Blueprint config + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> hdfsProps = new HashMap<String, String>(); + properties.put("hdfs-site", hdfsProps); + hdfsProps.put("foo", "val"); + hdfsProps.put("bar", "val"); + hdfsProps.put("secret", "SECRET:hdfs-site:1:test"); + + Map<String, String> category1Props = new HashMap<String, String>(); + properties.put("category1", category1Props); + category1Props.put("prop1", "val"); + + Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>(); + Configuration configuration = new Configuration(properties, attributes, EMPTY_CONFIGURATION); + // set config for group2 which contains a required property + + replay(stack, group1, group2); + + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration); + try { + blueprint.validateRequiredProperties(); + fail("Expected exception to be thrown for using secret reference"); + } catch (InvalidTopologyException e) { + System.out.println("****" + e.getMessage() + "***"); + } + + } + //todo: ensure coverage for these existing tests // private void validateEntity(BlueprintEntity entity, boolean containsConfig) { http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml index 1c02e86..f762116 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml @@ -24,6 +24,12 @@ <!-- ResourceManager --> <property> + <name>test.password</name> + <value> </value> + <property-type>PASSWORD</property-type> + </property> + + <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>localhost:8025</value> <deleted>true</deleted>