AMBARI-21180. Component command changes must include version numbers for all services (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5a1bd715 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5a1bd715 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5a1bd715 Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 5a1bd715e11b2e3a79b139118a90b66eca32f025 Parents: 12202a2 Author: Nate Cole <nc...@hortonworks.com> Authored: Mon Jun 5 17:39:42 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Tue Jun 6 10:33:14 2017 -0400 ---------------------------------------------------------------------- .../actionmanager/ExecutionCommandWrapper.java | 65 +++++++++++--------- .../ambari/server/agent/ExecutionCommand.java | 57 ++++++++++++----- .../AmbariCustomCommandExecutionHelper.java | 4 +- .../AmbariManagementControllerImpl.java | 2 - .../STORM/0.9.1/package/scripts/params_linux.py | 2 - .../1.0.1.3.0/package/scripts/params_linux.py | 2 - .../AmbariCustomCommandExecutionHelperTest.java | 11 ++-- 7 files changed, 84 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java index 2ec09d9..efd609a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java @@ -35,6 +35,7 @@ import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,7 +144,8 @@ public class ExecutionCommandWrapper { // now that the tags have been updated (if necessary), fetch the // configurations Map<String, Map<String, String>> configurationTags = executionCommand.getConfigurationTags(); - if (null != configurationTags && !configurationTags.isEmpty()) { + + if (MapUtils.isNotEmpty(configurationTags)) { Map<String, Map<String, String>> configProperties = configHelper .getEffectiveConfigProperties(cluster, configurationTags); @@ -182,43 +184,46 @@ public class ExecutionCommandWrapper { executionCommand.getConfigurationAttributes().get(type)); } } + } - // set the repository version for the component this command is for - - // always use the current desired version - try { - RepositoryVersionEntity repositoryVersion = null; - String serviceName = executionCommand.getServiceName(); - if (!StringUtils.isEmpty(serviceName)) { - Service service = cluster.getService(serviceName); - if (null != service) { - repositoryVersion = service.getDesiredRepositoryVersion(); - } + // set the repository version for the component this command is for - + // always use the current desired version + try { + RepositoryVersionEntity repositoryVersion = null; + String serviceName = executionCommand.getServiceName(); + if (!StringUtils.isEmpty(serviceName)) { + Service service = cluster.getService(serviceName); + if (null != service) { + repositoryVersion = service.getDesiredRepositoryVersion(); + } - String componentName = executionCommand.getComponentName(); - if (!StringUtils.isEmpty(componentName)) { - ServiceComponent serviceComponent = service.getServiceComponent( - executionCommand.getComponentName()); + String componentName = executionCommand.getComponentName(); + if (!StringUtils.isEmpty(componentName)) { + ServiceComponent serviceComponent = service.getServiceComponent( + executionCommand.getComponentName()); - if (null != serviceComponent) { - repositoryVersion = serviceComponent.getDesiredRepositoryVersion(); - } + if (null != serviceComponent) { + repositoryVersion = serviceComponent.getDesiredRepositoryVersion(); } } + } - if (null != repositoryVersion) { - executionCommand.getCommandParams().put(KeyNames.VERSION, - repositoryVersion.getVersion()); - executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION, - repositoryVersion.getVersion()); - } - } catch (ServiceNotFoundException serviceNotFoundException) { - // it's possible that there are commands specified for a service where - // the service doesn't exist yet - LOG.warn( - "The service {} is not installed in the cluster. No repository version will be sent for this command.", - executionCommand.getServiceName()); + if (null != repositoryVersion) { + executionCommand.getCommandParams().put(KeyNames.VERSION, repositoryVersion.getVersion()); + executionCommand.getHostLevelParams().put(KeyNames.CURRENT_VERSION, repositoryVersion.getVersion()); } + } catch (ServiceNotFoundException serviceNotFoundException) { + // it's possible that there are commands specified for a service where + // the service doesn't exist yet + LOG.warn( + "The service {} is not installed in the cluster. No repository version will be sent for this command.", + executionCommand.getServiceName()); } + + // set the desired versions of versionable components. This is safe even during an upgrade because + // we are "loading-late": components that have not yet upgraded in an EU will have the correct versions. + executionCommand.setComponentVersions(cluster); + } catch (ClusterNotFoundException cnfe) { // it's possible that there are commands without clusters; in such cases, // just return the de-serialized command and don't try to read configs http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 17b89b7..bd4c9dd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -26,8 +26,11 @@ import java.util.Set; import org.apache.ambari.annotations.Experimental; import org.apache.ambari.annotations.ExperimentalFeature; +import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.RoleCommand; -import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -110,9 +113,6 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("localComponents") private Set<String> localComponents = new HashSet<>(); - @SerializedName("availableServices") - private Map<String, String> availableServices = new HashMap<>(); - /** * "true" or "false" indicating whether this * service is enabled for credential store use. @@ -153,6 +153,9 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("repositoryFile") private CommandRepository commandRepository; + @SerializedName("componentVersionMap") + private Map<String, Map<String, String>> componentVersionMap = new HashMap<>(); + public void setConfigurationCredentials(Map<String, Map<String, String>> configurationCredentials) { this.configurationCredentials = configurationCredentials; } @@ -300,18 +303,6 @@ public class ExecutionCommand extends AgentCommand { this.localComponents = localComponents; } - public Map<String, String> getAvailableServices() { - return availableServices; - } - - public void setAvailableServicesFromServiceInfoMap(Map<String, ServiceInfo> serviceInfoMap) { - Map<String, String> serviceVersionMap = new HashMap<>(); - for (Map.Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) { - serviceVersionMap.put(entry.getKey(), entry.getValue().getVersion()); - } - availableServices = serviceVersionMap; - } - public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() { return configurationAttributes; } @@ -523,4 +514,38 @@ public class ExecutionCommand extends AgentCommand { comment = "This should be replaced by a map of all service component versions") String CURRENT_VERSION = "current_version"; } + + /** + * @return + */ + public Map<String, Map<String, String>> getComponentVersionMap() { + return componentVersionMap; + } + + /** + * Used to set a map of {service -> { component -> version}}. This is necessary when performing + * an upgrade to correct build paths of required binaries. + * @param cluster the cluster from which to build the map + */ + public void setComponentVersions(Cluster cluster) throws AmbariException { + Map<String, Map<String, String>> componentVersionMap = new HashMap<>(); + + for (Service service : cluster.getServices().values()) { + Map<String, String> componentMap = new HashMap<>(); + + boolean shouldSet = false; + for (ServiceComponent component : service.getServiceComponents().values()) { + if (component.isVersionAdvertised()) { + shouldSet = true; + componentMap.put(component.getName(), component.getDesiredVersion()); + } + } + + if (shouldSet) { + componentVersionMap.put(service.getName(), componentMap); + } + } + + this.componentVersionMap = componentVersionMap; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index cabb9d6..1f3b4b0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -384,12 +384,12 @@ public class AmbariCustomCommandExecutionHelper { execCmd.setCommandType(AgentCommandType.BACKGROUND_EXECUTION_COMMAND); } + execCmd.setComponentVersions(cluster); + execCmd.setConfigurations(configurations); execCmd.setConfigurationAttributes(configurationAttributes); execCmd.setConfigurationTags(configTags); - execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion())); - // Get the value of credential store enabled from the DB Service clusterService = cluster.getService(serviceName); execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled())); http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 186a19e..329998b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2555,11 +2555,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle cluster.addSuspendedUpgradeParameters(commandParams, roleParams); } - execCmd.setRoleParams(roleParams); execCmd.setCommandParams(commandParams); - execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion())); execCmd.setRepositoryFile(customCommandExecutionHelper.getCommandRepository(cluster, component, host)); hostParams.put(KeyNames.CURRENT_VERSION, repoVersion.getVersion()); http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py index 78ec165..664c582 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py @@ -394,8 +394,6 @@ if enable_ranger_storm: namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 -availableServices = config['availableServices'] - hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py index 78ec165..664c582 100644 --- a/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1.3.0/package/scripts/params_linux.py @@ -394,8 +394,6 @@ if enable_ranger_storm: namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 -availableServices = config['availableServices'] - hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None http://git-wip-us.apache.org/repos/asf/ambari/blob/5a1bd715/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java index a6c0c32..240517f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java @@ -70,7 +70,6 @@ import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.State; @@ -79,6 +78,7 @@ import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.collections.MapUtils; import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.EasyMockRule; @@ -553,14 +553,15 @@ public class AmbariCustomCommandExecutionHelperTest { ambariManagementController.createAction(actionRequest, requestProperties); StackId stackId = clusters.getCluster("c1").getDesiredStackVersion(); - Map<String, ServiceInfo> services = ambariManagementController.getAmbariMetaInfo().getServices(stackId.getStackName(), stackId.getStackVersion()); Request request = requestCapture.getValue(); Stage stage = request.getStages().iterator().next(); List<ExecutionCommandWrapper> commands = stage.getExecutionCommands("c1-c6401"); ExecutionCommand command = commands.get(0).getExecutionCommand(); - for (String service : services.keySet()) { - Assert.assertEquals(command.getAvailableServices().get(service), services.get(service).getVersion()); - } + + // ZK is the only service that is versionable + Assert.assertFalse(MapUtils.isEmpty(command.getComponentVersionMap())); + Assert.assertEquals(1, command.getComponentVersionMap().size()); + Assert.assertTrue(command.getComponentVersionMap().containsKey("ZOOKEEPER")); } @Test