Repository: ambari Updated Branches: refs/heads/branch-2.1 6971ed133 -> 3384fd917
Revert "AMBARI-13582. Allow use passwords references in custom actions. (vbrodetskyi via yusaku)" This reverts commit 64d7b1dd9b777849b40ee4128a8aa72a1b85a4a2. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3384fd91 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3384fd91 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3384fd91 Branch: refs/heads/branch-2.1 Commit: 3384fd9175b4055f497d926af6a4420857121bec Parents: 6971ed1 Author: Yusaku Sako <yus...@hortonworks.com> Authored: Wed Oct 28 13:10:16 2015 -0700 Committer: Yusaku Sako <yus...@hortonworks.com> Committed: Wed Oct 28 13:10:16 2015 -0700 ---------------------------------------------------------------------- .../controller/AmbariActionExecutionHelper.java | 4 - .../AmbariManagementControllerImpl.java | 27 +++---- .../ambari/server/controller/AmbariServer.java | 2 +- .../controller/ConfigurationResponse.java | 13 +++- .../internal/BlueprintResourceProvider.java | 49 +++--------- .../apache/ambari/server/state/StackInfo.java | 19 ----- .../server/state/cluster/ClusterImpl.java | 14 +++- .../server/topology/BlueprintValidatorImpl.java | 27 ------- .../ambari/server/utils/SecretReference.java | 78 +++++--------------- .../AmbariManagementControllerTest.java | 24 ++---- .../internal/BlueprintResourceProviderTest.java | 47 ++---------- .../server/topology/BlueprintImplTest.java | 63 +--------------- .../services/YARN/configuration/yarn-site.xml | 6 -- 13 files changed, 82 insertions(+), 291 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 215aca8..d834731 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -46,7 +46,6 @@ import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent; -import org.apache.ambari.server.utils.SecretReference; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -407,9 +406,6 @@ public class AmbariActionExecutionHelper { } roleParams.putAll(actionContext.getParameters()); - - SecretReference.replaceReferencesWithPasswords(roleParams, cluster); - if (componentInfo != null) { roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 2fb73ee..615b46d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -728,7 +728,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle String passwordPropertyValue = requestProperties.get(passwordProperty); if (!SecretReference.isSecret(passwordPropertyValue)) continue; - SecretReference ref = new SecretReference(passwordPropertyValue, cluster); + SecretReference ref = new SecretReference(passwordPropertyValue, passwordProperty, cluster); + if (!ref.getClusterName().equals(request.getClusterName())) + throw new AmbariException("Can not reference to different cluster in SECRET"); String refValue = ref.getValue(); requestProperties.put(passwordProperty, refValue); } @@ -929,9 +931,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (LOG.isDebugEnabled()) { LOG.debug("Received a getClusters request" - + ", clusterName=" + request.getClusterName() - + ", clusterId=" + request.getClusterId() - + ", stackInfo=" + request.getStackVersion()); + + ", clusterName=" + request.getClusterName() + + ", clusterId=" + request.getClusterId() + + ", stackInfo=" + request.getStackVersion()); } Cluster singleCluster = null; @@ -1398,7 +1400,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (propertiesTypes.containsKey(PropertyType.PASSWORD) && propertiesTypes.get(PropertyType.PASSWORD).contains(propertyName)) { if (SecretReference.isSecret(propertyValue)) { - SecretReference ref = new SecretReference(propertyValue, cluster); + SecretReference ref = new SecretReference(propertyValue, propertyName, cluster); requestConfigProperties.put(propertyName, ref.getValue()); } } @@ -3385,30 +3387,29 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle ExecuteCommandJson jsons = customCommandExecutionHelper.getCommandJson(actionExecContext, cluster, stackId); String commandParamsForStage = jsons.getCommandParamsForStage(); - Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, new TypeToken<Map<String, String>>() - {}.getType()); // Ensure that the specified requestContext (if any) is set as the request context if (!requestContext.isEmpty()) { requestStageContainer.setRequestContext(requestContext); } - // replace password references in requestProperties - SecretReference.replaceReferencesWithPasswords(commandParamsStage, cluster); - // If the request is to perform the Kerberos service check, set up the stages to // ensure that the (cluster-level) smoke user principal and keytab is available on all hosts boolean kerberosServiceCheck = Role.KERBEROS_SERVICE_CHECK.name().equals(actionRequest.getCommandName()); if (kerberosServiceCheck) { // Parse the command parameters into a map so that additional values may be added to it + Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, + new TypeToken<Map<String, String>>() { + }.getType()); try { requestStageContainer = kerberosHelper.createTestIdentity(cluster, commandParamsStage, requestStageContainer); } catch (KerberosOperationException e) { throw new IllegalArgumentException(e.getMessage(), e); } - } - commandParamsForStage = gson.toJson(commandParamsStage); + // Recreate commandParamsForStage with the added values + commandParamsForStage = gson.toJson(commandParamsStage); + } Stage stage = createNewStage(requestStageContainer.getLastStageId(), cluster, requestId, requestContext, jsons.getClusterHostInfo(), commandParamsForStage, jsons.getHostParamsForStage()); @@ -3440,7 +3441,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle if (kerberosServiceCheck) { // Parse the command parameters into a map so that existing values may be accessed and // additional values may be added to it. - commandParamsStage = gson.fromJson(commandParamsForStage, + Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, new TypeToken<Map<String, String>>() { }.getType()); http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index 57ae04c..021f4c8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -687,7 +687,7 @@ public class AmbariServer { StackDefinedPropertyProvider.init(injector); AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class)); BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class), - injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class), ambariMetaInfo); + injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class)); StackDependencyResourceProvider.init(ambariMetaInfo); ClusterResourceProvider.init(injector.getInstance(TopologyManager.class), injector.getInstance(TopologyRequestFactoryImpl.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java index eef3474..3ed9306 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java @@ -79,7 +79,7 @@ public class ConfigurationResponse { this.configs = configs; this.configAttributes = configAttributes; this.propertiesTypes = propertiesTypes; - SecretReference.replacePasswordsWithReferences(propertiesTypes, configs, type, version); + stubPasswords(); } /** @@ -215,4 +215,15 @@ public class ConfigurationResponse { public void setPropertiesTypes(Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes) { this.propertiesTypes = propertiesTypes; } + + private void stubPasswords(){ + if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD)) { + for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD)) { + if(configs.containsKey(pwdPropertyName)){ + String stub = SecretReference.generateStub(clusterName, type, version); + configs.put(pwdPropertyName, stub); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 5994094..6cb6a74 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -32,7 +32,6 @@ import java.util.Set; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.DuplicateResourceException; -import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.NoSuchResourceException; @@ -52,13 +51,11 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.stack.NoSuchStackException; -import org.apache.ambari.server.state.*; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; import org.apache.ambari.server.topology.InvalidTopologyException; import com.google.gson.Gson; -import org.apache.ambari.server.utils.SecretReference; /** @@ -119,10 +116,6 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide */ private static Gson jsonSerializer; - /** - * Used to get stack metainfo. - */ - private static AmbariMetaInfo ambariMetaInfo; // ----- Constructors ---------------------------------------------------- @@ -147,11 +140,10 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * @param dao blueprint data access object * @param gson json serializer */ - public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson, AmbariMetaInfo metaInfo) { + public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson) { blueprintFactory = factory; blueprintDAO = dao; jsonSerializer = gson; - ambariMetaInfo = metaInfo; } // ----- ResourceProvider ------------------------------------------------ @@ -183,7 +175,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide //todo: continue to use dao/entity directly or use blueprint factory? public Set<Resource> getResources(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, - NoSuchResourceException, NoSuchParentResourceException { + NoSuchResourceException, NoSuchParentResourceException { List<BlueprintEntity> results = null; boolean applyPredicate = false; @@ -250,8 +242,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide modifyResources(new Command<Void>() { @Override public Void invoke() throws AmbariException { - blueprintDAO.removeByName(blueprintName); - return null; + blueprintDAO.removeByName(blueprintName); + return null; } }); } @@ -271,7 +263,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * * @return a new resource instance for the given blueprint entity */ - protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) throws NoSuchResourceException { + protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) { StackEntity stackEntity = entity.getStack(); Resource resource = new ResourceImpl(Resource.Type.Blueprint); setResourceProperty(resource, BLUEPRINT_NAME_PROPERTY_ID, entity.getBlueprintName(), requestedIds); @@ -312,39 +304,16 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * @return list of configuration property maps */ List<Map<String, Map<String, Object>>> populateConfigurationList( - Collection<? extends BlueprintConfiguration> configurations) throws NoSuchResourceException { + Collection<? extends BlueprintConfiguration> configurations) { List<Map<String, Map<String, Object>>> listConfigurations = new ArrayList<Map<String, Map<String, Object>>>(); for (BlueprintConfiguration config : configurations) { Map<String, Map<String, Object>> mapConfigurations = new HashMap<String, Map<String, Object>>(); Map<String, Object> configTypeDefinition = new HashMap<String, Object>(); String type = config.getType(); - - if(config instanceof BlueprintConfigEntity) { - Map<String, String> properties = jsonSerializer.<Map<String, String>>fromJson( - config.getConfigData(), Map.class); - - StackEntity stack = ((BlueprintConfigEntity)config).getBlueprintEntity().getStack(); - StackInfo metaInfoStack; - - try { - metaInfoStack = ambariMetaInfo.getStack(stack.getStackName(), stack.getStackVersion()); - } catch (AmbariException e) { - throw new NoSuchResourceException(e.getMessage()); - } - - Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes = - metaInfoStack.getConfigPropertiesTypes(type); - - SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1l); - - configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); - } else { - Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson( - config.getConfigData(), Map.class); - configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); - } - + Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson( + config.getConfigData(), Map.class); + configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); Map<String, Map<String, String>> attributes = jsonSerializer.<Map<String, Map<String, String>>>fromJson( config.getConfigAttributes(), Map.class); if (attributes != null && !attributes.isEmpty()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index 2b9cd83..e3ac3e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -436,23 +436,4 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } return result; } - - public Map<PropertyInfo.PropertyType, Set<String>> getConfigPropertiesTypes(String configType) { - Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes = new HashMap<>(); - Collection<ServiceInfo> services = getServices(); - for (ServiceInfo serviceInfo : services) { - for (PropertyInfo propertyInfo : serviceInfo.getProperties()) { - if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty()) { - Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes(); - for (PropertyInfo.PropertyType propertyType : types) { - if (!propertiesTypes.containsKey(propertyType)) - propertiesTypes.put(propertyType, new HashSet<String>()); - propertiesTypes.get(propertyType).add(propertyInfo.getName()); - } - } - } - } - return propertiesTypes; - } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 7703bfd..4e37e14 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -2920,7 +2920,19 @@ public class ClusterImpl implements Cluster { try { StackId stackId = this.getCurrentStackVersion(); StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); - propertiesTypes = stackInfo.getConfigPropertiesTypes(configType); + Collection<ServiceInfo> services = stackInfo.getServices(); + for (ServiceInfo serviceInfo : services) { + for (PropertyInfo propertyInfo : serviceInfo.getProperties()) { + if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty()) { + Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes(); + for (PropertyInfo.PropertyType propertyType : types) { + if (!propertiesTypes.containsKey(propertyType)) + propertiesTypes.put(propertyType, new HashSet<String>()); + propertiesTypes.get(propertyType).add(propertyInfo.getName()); + } + } + } + } } catch (Exception e) { } http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index 1c293ee..9e8f163 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -21,7 +21,6 @@ package org.apache.ambari.server.topology; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyInfo; -import org.apache.ambari.server.utils.SecretReference; import org.apache.ambari.server.utils.VersionUtils; import java.util.Collection; @@ -84,32 +83,6 @@ public class BlueprintValidatorImpl implements BlueprintValidator { // we don't want to include default stack properties so we can't just use hostGroup full properties Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties(); - // we need to have real passwords, not references - if(clusterConfigurations != null) { - StringBuilder errorMessage = new StringBuilder(); - boolean containsSecretReferences = false; - for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) { - String configType = configEntry.getKey(); - if (configEntry.getValue() != null) { - for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) { - String propertyName = propertyEntry.getKey(); - String propertyValue = propertyEntry.getValue(); - if (propertyValue != null) { - if (SecretReference.isSecret(propertyValue)) { - errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n"); - containsSecretReferences = true; - } - } - } - } - } - if(containsSecretReferences) { - throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + - "replace following properties with real passwords:\n"+errorMessage.toString()); - } - } - - for (HostGroup hostGroup : blueprint.getHostGroups().values()) { Collection<String> processedServices = new HashSet<String>(); Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java index d801975..2b1aeae 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java @@ -21,36 +21,37 @@ package org.apache.ambari.server.utils; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.PropertyInfo; import java.util.Map; -import java.util.Set; public class SecretReference { - private static final String secretPrefix = "SECRET"; + private String clusterName; private String configType; private Long version; private String value; + private String reference; - public SecretReference(String reference, Cluster cluster) throws AmbariException{ + public SecretReference(String reference, String propertyName, Cluster cluster) throws AmbariException{ String[] values = reference.split(":"); - - configType = values[1]; - version = Long.valueOf(values[2]); - - String propertyName = values[3]; - String clusterName = cluster.getClusterName(); + clusterName = values[1]; + configType = values[2]; + version = Long.valueOf(values[3]); Config refConfig = cluster.getConfigByVersion(configType, version); if(refConfig == null) - throw new AmbariException(String.format("Error when parsing secret reference. Cluster: %s does not contain ConfigType: %s ConfigVersion: %s", - clusterName, configType, version)); + throw new AmbariException(String.format("Cluster: %s does not contain ConfigType: %s ConfigVersion: %s", + cluster.getClusterName(), configType, version)); Map<String, String> refProperties = refConfig.getProperties(); if(!refProperties.containsKey(propertyName)) - throw new AmbariException(String.format("Error when parsing secret reference. Cluster: %s ConfigType: %s ConfigVersion: %s does not contain property '%s'", - clusterName, configType, version, propertyName)); - + throw new AmbariException(String.format("Cluster: %s ConfigType: %s ConfigVersion: %s does not contain property '%s'", + cluster.getClusterName(), configType, version, propertyName)); this.value = refProperties.get(propertyName); + + this.reference = reference; + } + + public String getClusterName() { + return clusterName; } public void setConfigType(String configType) { @@ -67,51 +68,10 @@ public class SecretReference { public static boolean isSecret(String value) { String[] values = value.split(":"); - return values.length == 4 && values[0].equals(secretPrefix); - } - - public static String generateStub(String configType, Long configVersion, String propertyName) { - return secretPrefix + ":" + configType + ":" + configVersion.toString() + ":" + propertyName; - } - - /** - * Replace secret references with appropriate real passwords. - * @param targetMap map in which replacement will be performed - * @param cluster current cluster - * @throws AmbariException - */ - public static void replaceReferencesWithPasswords(Map<String, String> targetMap, Cluster cluster) - throws AmbariException { - if(cluster != null) { - for (Map.Entry<String, String> propertyValueEntry : targetMap.entrySet()) { - String key = propertyValueEntry.getKey(); - String value = propertyValueEntry.getValue(); - if (value != null && SecretReference.isSecret(value)) { - SecretReference ref = new SecretReference(value, cluster); - targetMap.put(key, ref.getValue()); - } - } - } + return values.length == 4 && values[0].equals("SECRET"); } - /** - * Replace real passwords with secret references - * @param propertiesTypes map with properties types - * @param propertiesMap map with properties in which replacement will be performed - * @param configType configuration type - * @param configVersion configuration version - */ - public static void replacePasswordsWithReferences(Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes, - Map<String, String> propertiesMap, - String configType, - Long configVersion){ - if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD)) { - for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD)) { - if(propertiesMap.containsKey(pwdPropertyName)){ - String stub = SecretReference.generateStub(configType, configVersion, pwdPropertyName); - propertiesMap.put(pwdPropertyName, stub); - } - } - } + public static String generateStub(String clusterName, String configType, Long configVersion) { + return "SECRET:" + clusterName + ":" + configType + ":" + configVersion.toString(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 394bae4..f112f50 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -4123,7 +4123,7 @@ public class AmbariManagementControllerTest { Config config1 = cf.createNew(cluster, "global", new HashMap<String, String>() {{ put("key1", "value1"); - }}, new HashMap<String, Map<String, String>>()); + }}, new HashMap<String, Map<String,String>>()); config1.setTag("version1"); Config config2 = cf.createNew(cluster, "core-site", @@ -4132,15 +4132,8 @@ public class AmbariManagementControllerTest { }}, new HashMap<String, Map<String,String>>()); config2.setTag("version1"); - Config config3 = cf.createNew(cluster, "yarn-site", - new HashMap<String, String>() {{ - put("test.password", "supersecret"); - }}, new HashMap<String, Map<String,String>>()); - config3.setTag("version1"); - cluster.addConfig(config1); cluster.addConfig(config2); - cluster.addConfig(config3); Service hdfs = cluster.addService("HDFS"); hdfs.persist(); @@ -4168,7 +4161,6 @@ public class AmbariManagementControllerTest { Map<String, String> params = new HashMap<String, String>() {{ put("test", "test"); - put("pwd", "SECRET:yarn-site:1:test.password"); }}; Map<String, String> requestProperties = new HashMap<String, String>(); @@ -4201,8 +4193,6 @@ public class AmbariManagementControllerTest { Map<String, String> commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); Assert.assertTrue(commandParametersStage.containsKey("test")); - Assert.assertTrue(commandParametersStage.containsKey("pwd")); - Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret"); Assert.assertEquals("HDFS", cmd.getServiceName()); Assert.assertEquals("DATANODE", cmd.getComponentName()); Assert.assertNotNull(hostParametersStage.get("jdk_location")); @@ -4243,8 +4233,6 @@ public class AmbariManagementControllerTest { commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type); Assert.assertTrue(commandParametersStage.containsKey("test")); - Assert.assertTrue(commandParametersStage.containsKey("pwd")); - Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret"); Assert.assertEquals("HDFS", cmd.getServiceName()); Assert.assertEquals("DATANODE", cmd.getComponentName()); Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext()); @@ -10604,7 +10592,7 @@ public class AmbariManagementControllerTest { "hdfs-site", "version2", new HashMap<String, String>(){{ - put("test.password", "SECRET:hdfs-site:1:test.password"); + put("test.password", "SECRET:c1:hdfs-site:1"); put("new", "new");//need this to mark config as "changed" }}, new HashMap<String, Map<String, String>>() @@ -10629,7 +10617,7 @@ public class AmbariManagementControllerTest { "hdfs-site", "version3", new HashMap<String, String>(){{ - put("test.password", "SECRET:hdfs-site:666:test.password"); + put("test.password", "SECRET:c1:hdfs-site:666"); }}, new HashMap<String, Map<String, String>>() ); @@ -10657,7 +10645,7 @@ public class AmbariManagementControllerTest { "hdfs-site", "version5", new HashMap<String, String>(){{ - put("test.password", "SECRET:hdfs-site:4:test.password"); + put("test.password", "SECRET:c1:hdfs-site:4"); put("new", "new"); }}, new HashMap<String, Map<String, String>>() @@ -10668,7 +10656,7 @@ public class AmbariManagementControllerTest { controller.updateClusters(Collections.singleton(crReq), null); fail("Request need to be failed with wrong secret reference"); } catch (AmbariException e) { - assertEquals("Error when parsing secret reference. Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain property 'test.password'", + assertEquals("Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain property 'test.password'", e.getMessage()); } cl.getAllConfigs(); @@ -10691,7 +10679,7 @@ public class AmbariManagementControllerTest { add(configRequest); }}); for(ConfigurationResponse resp : requestedConfigs) { - String secretName = "SECRET:hdfs-site:"+resp.getVersion().toString()+":test.password"; + String secretName = "SECRET:foo1:hdfs-site:"+resp.getVersion().toString(); if(resp.getConfigs().containsKey("test.password")) { assertEquals(resp.getConfigs().get("test.password"), secretName); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java index 8ef4bbb..5bfdebb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategy; @@ -70,8 +69,6 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.orm.entities.StackEntity; -import org.apache.ambari.server.state.*; -import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; @@ -107,7 +104,7 @@ public class BlueprintResourceProviderTest { @BeforeClass public static void initClass() { - BlueprintResourceProvider.init(blueprintFactory, dao, gson, metaInfo); + BlueprintResourceProvider.init(blueprintFactory, dao, gson); StackEntity stackEntity = new StackEntity(); stackEntity.setStackName("test-stack-name"); @@ -384,12 +381,7 @@ public class BlueprintResourceProviderTest { @Test public void testGetResourcesNoPredicate_withConfiguration() throws SystemException, UnsupportedPropertyException, - NoSuchParentResourceException, NoSuchResourceException, AmbariException { - - StackInfo info = createMock(StackInfo.class); - expect(info.getConfigPropertiesTypes("core-site")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes(); - expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes(); - replay(info, metaInfo); + NoSuchParentResourceException, NoSuchResourceException { Request request = createNiceMock(Request.class); Set<Map<String, Object>> testProperties = getBlueprintTestProperties(); @@ -959,44 +951,20 @@ public class BlueprintResourceProviderTest { @Test public void testPopulateConfigurationList() throws Exception { - StackEntity stackEntity = new StackEntity(); - stackEntity.setStackName("test-stack-name"); - stackEntity.setStackVersion("test-stack-version"); - BlueprintEntity entity = createMock(BlueprintEntity.class); - expect(entity.getStack()).andReturn(stackEntity).anyTimes(); - - HashMap<PropertyInfo.PropertyType, Set<String>> pwdProperties = new HashMap<PropertyInfo.PropertyType, Set<String>>() {{ - put(PropertyInfo.PropertyType.PASSWORD, new HashSet<String>(){{ - add("test.password"); - }}); - }}; - - StackInfo info = createMock(StackInfo.class); - expect(info.getConfigPropertiesTypes("type1")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes(); - expect(info.getConfigPropertiesTypes("type2")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes(); - expect(info.getConfigPropertiesTypes("type3")).andReturn(pwdProperties).anyTimes(); - expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes(); - - replay(info, metaInfo, entity); - - // attributes is null - BlueprintConfigEntity config1 = new BlueprintConfigEntity(); + BlueprintConfiguration config1 = new BlueprintConfigEntity(); config1.setType("type1"); config1.setConfigData("{\"key1\":\"value1\"}"); - config1.setBlueprintEntity(entity); // attributes is empty - BlueprintConfigEntity config2 = new BlueprintConfigEntity(); + BlueprintConfiguration config2 = new BlueprintConfigEntity(); config2.setType("type2"); config2.setConfigData("{\"key2\":\"value2\"}"); config2.setConfigAttributes("{}"); - config2.setBlueprintEntity(entity); // attributes is provided - BlueprintConfigEntity config3 = new BlueprintConfigEntity(); + BlueprintConfiguration config3 = new BlueprintConfigEntity(); config3.setType("type3"); - config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\",\"test.password\":\"pwdValue\"}"); + config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\"}"); config3.setConfigAttributes("{\"final\":{\"key3\":\"attrValue1\",\"key4\":\"attrValue2\"}}"); - config3.setBlueprintEntity(entity); List<Map<String, Map<String, Object>>> configs = provider.populateConfigurationList(Arrays.asList(config1, config2, config3)); @@ -1042,10 +1010,9 @@ public class BlueprintResourceProviderTest { Map<String, String> confProperties3 = (Map<String, String>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_PROPERTY_ID); assertNotNull(confProperties3); - assertEquals(3, confProperties3.size()); + assertEquals(2, confProperties3.size()); assertEquals("value3", confProperties3.get("key3")); assertEquals("value4", confProperties3.get("key4")); - assertEquals("SECRET:type3:-1:test.password", confProperties3.get("test.password")); assertTrue(typeConfig3.containsKey(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID)); Map<String, Map<String, String>> confAttributes3 = (Map<String, Map<String, String>>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID); http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index de740f4..9d4163a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -183,6 +183,7 @@ public class BlueprintImplTest { properties.put("hdfs-site", hdfsProps); hdfsProps.put("foo", "val"); hdfsProps.put("bar", "val"); + Map<String, String> category1Props = new HashMap<String, String>(); properties.put("category1", category1Props); category1Props.put("prop1", "val"); @@ -281,68 +282,6 @@ public class BlueprintImplTest { verify(stack, group1, group2); } - @Test - public void testValidateConfigurations__secretReference(){ - Stack stack = createNiceMock(Stack.class); - - HostGroup group1 = createNiceMock(HostGroup.class); - HostGroup group2 = createNiceMock(HostGroup.class); - Collection<HostGroup> hostGroups = new HashSet<HostGroup>(); - hostGroups.add(group1); - hostGroups.add(group2); - - Set<String> group1Components = new HashSet<String>(); - group1Components.add("c1"); - group1Components.add("c2"); - - Set<String> group2Components = new HashSet<String>(); - group2Components.add("c1"); - group2Components.add("c3"); - - Map<String, Map<String, String>> group2Props = new HashMap<String, Map<String, String>>(); - Map<String, String> group2Category2Props = new HashMap<String, String>(); - group2Props.put("category2", group2Category2Props); - group2Category2Props.put("prop2", "val"); - - Collection<Stack.ConfigProperty> requiredHDFSProperties = new HashSet<Stack.ConfigProperty>(); - requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "foo", null)); - requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "bar", null)); - requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "some_password", null)); - - requiredHDFSProperties.add(new Stack.ConfigProperty("category1", "prop1", null)); - - Collection<Stack.ConfigProperty> requiredService2Properties = new HashSet<Stack.ConfigProperty>(); - requiredService2Properties.add(new Stack.ConfigProperty("category2", "prop2", null)); - - - // Blueprint config - Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); - Map<String, String> hdfsProps = new HashMap<String, String>(); - properties.put("hdfs-site", hdfsProps); - hdfsProps.put("foo", "val"); - hdfsProps.put("bar", "val"); - hdfsProps.put("secret", "SECRET:hdfs-site:1:test"); - - Map<String, String> category1Props = new HashMap<String, String>(); - properties.put("category1", category1Props); - category1Props.put("prop1", "val"); - - Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>(); - Configuration configuration = new Configuration(properties, attributes, EMPTY_CONFIGURATION); - // set config for group2 which contains a required property - - replay(stack, group1, group2); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration); - try { - blueprint.validateRequiredProperties(); - fail("Expected exception to be thrown for using secret reference"); - } catch (InvalidTopologyException e) { - System.out.println("****" + e.getMessage() + "***"); - } - - } - //todo: ensure coverage for these existing tests // private void validateEntity(BlueprintEntity entity, boolean containsConfig) { http://git-wip-us.apache.org/repos/asf/ambari/blob/3384fd91/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml index f762116..1c02e86 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml @@ -24,12 +24,6 @@ <!-- ResourceManager --> <property> - <name>test.password</name> - <value> </value> - <property-type>PASSWORD</property-type> - </property> - - <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>localhost:8025</value> <deleted>true</deleted>