AMBARI-21617. Function ru_set_all is not working correctly due to changed command structure (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/abb9b463 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/abb9b463 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/abb9b463 Branch: refs/heads/branch-2.6 Commit: abb9b4638772c47fcdbeffde47cfed88460757f6 Parents: b07dc77 Author: Nate Cole <nc...@hortonworks.com> Authored: Mon Jul 31 14:31:37 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Tue Aug 1 09:12:18 2017 -0400 ---------------------------------------------------------------------- .../ambari/server/actionmanager/Stage.java | 28 +- .../internal/UpgradeResourceProvider.java | 17 ++ .../ambari/server/state/ConfigHelper.java | 1 - .../ambari/server/state/UpgradeContext.java | 2 +- .../ambari/server/state/UpgradeHelper.java | 41 ++- .../custom_actions/scripts/ru_set_all.py | 38 --- .../ExecutionCommandWrapperTest.java | 4 +- .../server/api/services/AmbariMetaInfoTest.java | 14 +- .../AmbariManagementControllerTest.java | 4 +- .../BackgroundCustomCommandExecutionTest.java | 2 +- ...hYarnCapacitySchedulerReleaseConfigTest.java | 2 +- .../internal/UpgradeResourceProviderTest.java | 6 + .../ambari/server/state/UpgradeHelperTest.java | 83 +++++- .../server/topology/AmbariContextTest.java | 58 ++-- .../server/upgrade/UpgradeCatalogTest.java | 2 +- .../python/custom_actions/test_ru_set_all.py | 47 +--- .../upgrades/upgrade_execute_task_test.xml | 42 +++ .../HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml | 267 +++++++++++++++++++ 18 files changed, 517 insertions(+), 141 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java index a01313e..1d6ceaf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/Stage.java @@ -19,7 +19,6 @@ package org.apache.ambari.server.actionmanager; import java.util.ArrayList; import java.util.Collection; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -34,7 +33,6 @@ import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.agent.AgentCommand.AgentCommandType; import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.metadata.RoleCommandOrder; import org.apache.ambari.server.metadata.RoleCommandPair; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; import org.apache.ambari.server.orm.entities.HostRoleCommandEntity; @@ -89,13 +87,13 @@ public class Stage { private volatile boolean wrappersLoaded = false; //Map of roles to successFactors for this stage. Default is 1 i.e. 100% - private Map<Role, Float> successFactors = new HashMap<Role, Float>(); + private Map<Role, Float> successFactors = new HashMap<>(); //Map of host to host-roles Map<String, Map<String, HostRoleCommand>> hostRoleCommands = - new TreeMap<String, Map<String, HostRoleCommand>>(); + new TreeMap<>(); private Map<String, List<ExecutionCommandWrapper>> commandsToSend = - new TreeMap<String, List<ExecutionCommandWrapper>>(); + new TreeMap<>(); @Inject private HostRoleCommandFactory hostRoleCommandFactory; @@ -192,7 +190,9 @@ public class Stage { stageEntity.setHostRoleCommands(new ArrayList<HostRoleCommandEntity>()); stageEntity.setRoleSuccessCriterias(new ArrayList<RoleSuccessCriteriaEntity>()); stageEntity.setCommandParamsStage(commandParamsStage); - stageEntity.setHostParamsStage(hostParamsStage); + if (null != hostParamsStage) { + stageEntity.setHostParamsStage(hostParamsStage); + } stageEntity.setCommandExecutionType(commandExecutionType); for (Role role : successFactors.keySet()) { @@ -228,7 +228,7 @@ public class Stage { } public List<HostRoleCommand> getOrderedHostRoleCommands() { - List<HostRoleCommand> commands = new ArrayList<HostRoleCommand>(); + List<HostRoleCommand> commands = new ArrayList<>(); //Correct due to ordered maps for (Map.Entry<String, Map<String, HostRoleCommand>> hostRoleCommandEntry : hostRoleCommands.entrySet()) { for (Map.Entry<String, HostRoleCommand> roleCommandEntry : hostRoleCommandEntry.getValue().entrySet()) { @@ -457,7 +457,7 @@ public class Stage { ExecutionCommand cmd = commandWrapper.getExecutionCommand(); - Map<String, String> cmdParams = new HashMap<String, String>(); + Map<String, String> cmdParams = new HashMap<>(); if (commandParams != null) { cmdParams.putAll(commandParams); } @@ -466,18 +466,18 @@ public class Stage { } cmd.setCommandParams(cmdParams); - Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>(); + Map<String, Map<String, String>> configurations = new TreeMap<>(); cmd.setConfigurations(configurations); - Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>(); + Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<>(); cmd.setConfigurationAttributes(configurationAttributes); if (configTags == null) { - configTags = new TreeMap<String, Map<String, String>>(); + configTags = new TreeMap<>(); } cmd.setConfigurationTags(configTags); - Map<String, String> roleParams = new HashMap<String, String>(); + Map<String, String> roleParams = new HashMap<>(); roleParams.put(ServerAction.ACTION_NAME, actionName); if (userName != null) { roleParams.put(ServerAction.ACTION_USER_NAME, userName); @@ -505,7 +505,7 @@ public class Stage { Assert.notEmpty(cancelTargets, "Provided targets task Id are empty."); - Map<String, String> roleParams = new HashMap<String, String>(); + Map<String, String> roleParams = new HashMap<>(); roleParams.put("cancelTaskIdTargets", StringUtils.join(cancelTargets, ',')); cmd.setRoleParams(roleParams); @@ -516,7 +516,7 @@ public class Stage { * @return list of hosts */ public synchronized List<String> getHosts() { // TODO: Check whether method should be synchronized - List<String> hlist = new ArrayList<String>(); + List<String> hlist = new ArrayList<>(); for (String h : hostRoleCommands.keySet()) { hlist.add(h); } http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index c26f062..83a9510 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -99,6 +99,7 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper; import org.apache.ambari.server.state.stack.upgrade.Task; import org.apache.ambari.server.state.stack.upgrade.TaskWrapper; import org.apache.ambari.server.state.stack.upgrade.UpdateStackGrouping; +import org.apache.ambari.server.state.stack.upgrade.UpgradeScope; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent; import org.apache.ambari.server.utils.StageUtils; @@ -947,6 +948,22 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider Map<String, String> params = getNewParameterMap(request, context); params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks()); + // !!! when not scoped to a component (generic execution task) + if (context.isScoped(UpgradeScope.COMPLETE) && null == componentName) { + if (context.getDirection().isUpgrade()) { + params.put(KeyNames.VERSION, context.getRepositoryVersion().getVersion()); + } else { + // !!! in a full downgrade, the target version should be any of the history versions + UpgradeEntity lastUpgrade = s_upgradeDAO.findLastUpgradeForCluster( + cluster.getClusterId(), Direction.UPGRADE); + + @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES, + comment = "Shouldn't be getting the overall downgrade-to version.") + UpgradeHistoryEntity lastHistory = lastUpgrade.getHistory().iterator().next(); + params.put(KeyNames.VERSION, lastHistory.getFromReposistoryVersion().getVersion()); + } + } + // Apply additional parameters to the command that come from the stage. applyAdditionalParameters(wrapper, params); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 8270370..16fc971 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -1110,7 +1110,6 @@ public class ConfigHelper { serviceMapped.put(service, new HashSet<Config>()); } serviceMapped.get(service).add(baseConfig); - } catch (Exception e) { // !!! ignore } http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index 6f469e5..8bf0400 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -669,7 +669,7 @@ public class UpgradeContext { * <p/> * If the direction is {@link Direction#UPGRADE} then this will return the * target repository which every service will be on if the upgrade is - * finalized. <br/> + * finalized. <p/> * If the direction is {@link Direction#DOWNGRADE} then this will return the * repository from which the downgrade is coming from. * http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index a5881d2..a5b40ff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; @@ -244,6 +245,8 @@ public class UpgradeHelper { if (StringUtils.isNotEmpty(preferredUpgradePackName) && packs.containsKey(preferredUpgradePackName)) { pack = packs.get(preferredUpgradePackName); + + LOG.warn("Upgrade pack '{}' not found for stack {}", preferredUpgradePackName, currentStack); } // Best-attempt at picking an upgrade pack assuming within the same stack whose target stack version matches. @@ -863,8 +866,11 @@ public class UpgradeHelper { String userName = controller.getAuthName(); Set<String> servicesInUpgrade = upgradeContext.getSupportedServices(); + Set<String> clusterConfigTypes = new HashSet<>(); + Set<String> processedClusterConfigTypes = new HashSet<>(); + // merge or revert configurations for any service that needs it - for( String serviceName : servicesInUpgrade ){ + for (String serviceName : servicesInUpgrade) { RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName); RepositoryVersionEntity targetRepositoryVersion = upgradeContext.getTargetRepositoryVersion(serviceName); StackId sourceStackId = sourceRepositoryVersion.getStackId(); @@ -901,6 +907,12 @@ public class UpgradeHelper { Map<String, Map<String, String>> newServiceDefaultConfigsByType = configHelper.getDefaultProperties( targetStackId, serviceName); + if (null == oldServiceDefaultConfigsByType || null == newServiceDefaultConfigsByType) { + continue; + } + + Set<String> foundConfigTypes = new HashSet<>(); + // find the current, existing configurations for the service List<Config> existingServiceConfigs = new ArrayList<>(); List<ServiceConfigEntity> latestServiceConfigs = m_serviceConfigDAO.getLastServiceConfigsForService( @@ -910,8 +922,23 @@ public class UpgradeHelper { List<ClusterConfigEntity> existingConfigurations = serviceConfig.getClusterConfigEntities(); for (ClusterConfigEntity currentServiceConfig : existingConfigurations) { String configurationType = currentServiceConfig.getType(); + Config currentClusterConfigForService = cluster.getDesiredConfigByType(configurationType); existingServiceConfigs.add(currentClusterConfigForService); + foundConfigTypes.add(configurationType); + } + } + + // !!! these are the types that come back from the config helper, but are not part of the service. + @SuppressWarnings("unchecked") + Set<String> missingConfigTypes = new HashSet<>(CollectionUtils.subtract(oldServiceDefaultConfigsByType.keySet(), + foundConfigTypes)); + + for (String missingConfigType : missingConfigTypes) { + Config config = cluster.getDesiredConfigByType(missingConfigType); + if (null != config) { + existingServiceConfigs.add(config); + clusterConfigTypes.add(missingConfigType); } } @@ -1013,8 +1040,18 @@ public class UpgradeHelper { } if (null != newServiceDefaultConfigsByType) { + + for (String clusterConfigType : clusterConfigTypes) { + if (processedClusterConfigTypes.contains(clusterConfigType)) { + newServiceDefaultConfigsByType.remove(clusterConfigType); + } else { + processedClusterConfigTypes.add(clusterConfigType); + } + + } + Set<String> configTypes = newServiceDefaultConfigsByType.keySet(); - LOG.info("The upgrade will create the following configurations for stack {}: {}", + LOG.warn("The upgrade will create the following configurations for stack {}: {}", targetStackId, StringUtils.join(configTypes, ',')); String serviceVersionNote = String.format("%s %s %s", direction.getText(true), http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py index a7732d9..e514c4b 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py @@ -117,44 +117,6 @@ class UpgradeSetAll(Script): stack_name, downgrade_from_version, downgrade_to_version)) return - # iterate through all directory conf mappings and undo the symlinks - for key, value in conf_select.get_package_dirs().iteritems(): - for directory_mapping in value: - original_config_directory = directory_mapping['conf_dir'] - self._unlink_config(original_config_directory) - - - def _unlink_config(self, original_conf_directory): - """ - Reverses the work performed in link_config. This should only be used when downgrading from - HDP 2.3 to 2.2 in order to undo the conf symlink work required for 2.3. - - 1. Checks if conf.backup exists, if not then do no work - 2. Check for existance of 'etc' symlink and remove it - 3. Rename conf.back back to conf - - :original_conf_directory: the original conf directory that was made into a symlink (/etc/component/conf) - """ - # calculate the parent and backup directories - original_conf_parent_directory = os.path.abspath(os.path.join(original_conf_directory, os.pardir)) - backup_conf_directory = os.path.join(original_conf_parent_directory, "conf.backup") - Logger.info("Analyzing potential link {0}".format(original_conf_directory)) - - if os.path.islink(original_conf_directory): - # remove the old symlink - Execute(("rm", original_conf_directory), sudo=True) - elif os.path.isdir(original_conf_directory): - Directory(original_conf_directory, action="delete") - else: - Logger.info(" Skipping the unlink of {0}; it is not a symlink or does not exist".format(original_conf_directory)) - - if os.path.isdir(backup_conf_directory): - # rename the backup to the original name - Logger.info(" Unlinking {0} and restoring {1}".format(original_conf_directory, backup_conf_directory)) - Execute(("mv", backup_conf_directory, original_conf_directory), sudo=True) - else: - Logger.info(" Skipping restoring config from backup {0} since it does not exist".format(backup_conf_directory)) - def is_host_skippable(stack_selector_path, formatted_version): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java index e20b7b3..f68f639 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java @@ -43,6 +43,7 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ConfigFactory; import org.apache.ambari.server.state.ConfigHelper; +import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent; import org.apache.ambari.server.utils.StageUtils; @@ -294,7 +295,8 @@ public class ExecutionCommandWrapperTest { StackId stackId = cluster.getDesiredStackVersion(); RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(stackId, "0.1-0000"); - cluster.addService("HDFS", repositoryVersion); + Service service = cluster.getService("HDFS"); + service.setDesiredRepositoryVersion(repositoryVersion); // first try with an INSTALL command - this should not populate version info ExecutionCommand executionCommand = new ExecutionCommand(); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 229b864..63cb21f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -92,6 +92,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -226,6 +227,7 @@ public class AmbariMetaInfoTest { } @Test + @Ignore public void testGetRepositoryDefault() throws Exception { // Scenario: user has internet and does nothing to repos via api // use the latest @@ -1714,7 +1716,7 @@ public class AmbariMetaInfoTest { } } - assertEquals(3, hostAlertCount); + assertEquals(2, hostAlertCount); assertEquals(10, definitions.size() - hostAlertCount); for (AlertDefinitionEntity definition : definitions) { @@ -1725,7 +1727,7 @@ public class AmbariMetaInfoTest { metaInfo.reconcileAlertDefinitions(clusters, false); definitions = dao.findAll(); - assertEquals(13, definitions.size()); + assertEquals(12, definitions.size()); for (AlertDefinitionEntity definition : definitions) { assertEquals(28, definition.getScheduleInterval().intValue()); @@ -1734,7 +1736,7 @@ public class AmbariMetaInfoTest { // find all enabled for the cluster should find 6 (the ones from HDFS; // it will not find the agent alert since it's not bound to the cluster) definitions = dao.findAllEnabled(cluster.getClusterId()); - assertEquals(12, definitions.size()); + assertEquals(11, definitions.size()); // create new definition AlertDefinitionEntity entity = new AlertDefinitionEntity(); @@ -1753,19 +1755,19 @@ public class AmbariMetaInfoTest { // verify the new definition is found (6 HDFS + 1 new one) definitions = dao.findAllEnabled(cluster.getClusterId()); - assertEquals(13, definitions.size()); + assertEquals(12, definitions.size()); // reconcile, which should disable our bad definition metaInfo.reconcileAlertDefinitions(clusters, false); // find all enabled for the cluster should find 6 definitions = dao.findAllEnabled(cluster.getClusterId()); - assertEquals(12, definitions.size()); + assertEquals(11, definitions.size()); // find all should find 6 HDFS + 1 disabled + 1 agent alert + 2 server // alerts definitions = dao.findAll(); - assertEquals(14, definitions.size()); + assertEquals(13, definitions.size()); entity = dao.findById(entity.getDefinitionId()); assertFalse(entity.getEnabled()); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 011214c..59fa7f7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -2253,7 +2253,7 @@ public class AmbariManagementControllerTest { resp = controller.getClusters(Collections.singleton(r)); Assert.assertTrue(resp.size() >= 3); - r = new ClusterRequest(null, null, "", null); + r = new ClusterRequest(null, null, null, null); resp = controller.getClusters(Collections.singleton(r)); Assert.assertTrue("Stack ID request is invalid and expect them all", resp.size() > 3); } @@ -6978,7 +6978,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(1, responsesWithParams.size()); StackVersionResponse resp = responsesWithParams.iterator().next(); assertNotNull(resp.getUpgradePacks()); - assertEquals(14, resp.getUpgradePacks().size()); + assertEquals(15, resp.getUpgradePacks().size()); assertTrue(resp.getUpgradePacks().contains("upgrade_test")); } http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java index 4c237a7..fa9475e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java @@ -218,7 +218,7 @@ public class BackgroundCustomCommandExecutionTest { dStateStr = desiredState.toString(); } ServiceRequest r1 = new ServiceRequest(clusterName, serviceName, - m_repositoryVersion.getId(), m_repositoryVersion.getVersion(), dStateStr); + m_repositoryVersion.getId(), dStateStr); Set<ServiceRequest> requests = new HashSet<>(); requests.add(r1); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java index 122e4ef..5755e7e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java @@ -214,7 +214,7 @@ public class RefreshYarnCapacitySchedulerReleaseConfigTest { new StackId("HDP-2.0.7"), "2.0.7-1234"); ServiceRequest r1 = new ServiceRequest(clusterName, serviceName, - repositoryVersion.getId(), repositoryVersion.getVersion(), dStateStr); + repositoryVersion.getId(), dStateStr); Set<ServiceRequest> requests = new HashSet<>(); requests.add(r1); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java index c5fc039..aa216f7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java @@ -1601,6 +1601,12 @@ public class UpgradeResourceProviderTest extends EasyMockSupport { public void testCreatePatchRevertUpgrade() throws Exception { Cluster cluster = clusters.getCluster("c1"); + // add a single ZK server and client on 2.1.1.0 + Service service = cluster.addService("HBASE", repoVersionEntity2110); + ServiceComponent component = service.addServiceComponent("HBASE_MASTER"); + ServiceComponentHost sch = component.addServiceComponentHost("h1"); + sch.setVersion("2.1.1.0"); + File f = new File("src/test/resources/hbase_version_test.xml"); repoVersionEntity2112.setVersionXml(IOUtils.toString(new FileInputStream(f))); repoVersionEntity2112.setVersionXsd("version_definition.xsd"); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index f0b884b..9b85242 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -49,6 +49,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ClusterRequest; import org.apache.ambari.server.controller.ConfigurationRequest; +import org.apache.ambari.server.controller.internal.UpgradeResourceProvider; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.orm.OrmTestHelper; @@ -92,6 +93,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.security.core.context.SecurityContextHolder; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.gson.Gson; @@ -333,7 +335,7 @@ public class UpgradeHelperTest extends EasyMockSupport { assertEquals("Save Cluster State", postGroup.items.get(1).getText()); assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType()); - assertEquals(2, groups.get(0).items.size()); + assertEquals(3, groups.get(0).items.size()); assertEquals(7, groups.get(1).items.size()); assertEquals(2, groups.get(2).items.size()); @@ -719,7 +721,6 @@ public class UpgradeHelperTest extends EasyMockSupport { List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); - System.out.println(">> transfers"+transfers); assertEquals(6, transfers.size()); assertEquals("copy-key", transfers.get(0).fromKey); @@ -797,8 +798,6 @@ public class UpgradeHelperTest extends EasyMockSupport { new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); - System.out.println(" testConfigTaskConditionMet >> transfer"+transfers); - assertEquals("copy-key-one", transfers.get(0).fromKey); assertEquals("copy-to-key-one", transfers.get(0).toKey); @@ -2425,6 +2424,82 @@ public class UpgradeHelperTest extends EasyMockSupport { assertEquals("three-changed", expectedBazType.get("3")); } + @Test + public void testMergeConfigurationsWithClusterEnv() throws Exception { + Cluster cluster = makeCluster(true); + + StackId oldStack = cluster.getDesiredStackVersion(); + StackId newStack = new StackId("HDP-2.5.0"); + + ConfigFactory cf = injector.getInstance(ConfigFactory.class); + + Config clusterEnv = cf.createNew(cluster, "cluster-env", "version1", + ImmutableMap.<String, String>builder().put("a", "b").build(), + Collections.<String, Map<String, String>>emptyMap()); + + Config zooCfg = cf.createNew(cluster, "zoo.cfg", "version1", + ImmutableMap.<String, String>builder().put("c", "d").build(), + Collections.<String, Map<String, String>>emptyMap()); + + cluster.addDesiredConfig("admin", Sets.newHashSet(clusterEnv, zooCfg)); + + Map<String, Map<String, String>> stackMap = new HashMap<>(); + stackMap.put("cluster-env", new HashMap<String, String>()); + stackMap.put("hive-site", new HashMap<String, String>()); + + final Map<String, String> clusterEnvMap = new HashMap<>(); + + Capture<Cluster> captureCluster = Capture.newInstance(); + Capture<StackId> captureStackId = Capture.newInstance(); + Capture<AmbariManagementController> captureAmc = Capture.newInstance(); + + Capture<Map<String, Map<String, String>>> cap = new Capture<Map<String, Map<String, String>>>() { + @Override + public void setValue(Map<String, Map<String, String>> value) { + if (value.containsKey("cluster-env")) { + clusterEnvMap.putAll(value.get("cluster-env")); + } + } + }; + + Capture<String> captureUsername = Capture.newInstance(); + Capture<String> captureNote = Capture.newInstance(); + + EasyMock.reset(m_configHelper); + expect(m_configHelper.getDefaultProperties(oldStack, "HIVE")).andReturn(stackMap).atLeastOnce(); + expect(m_configHelper.getDefaultProperties(newStack, "HIVE")).andReturn(stackMap).atLeastOnce(); + expect(m_configHelper.getDefaultProperties(oldStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce(); + expect(m_configHelper.getDefaultProperties(newStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce(); + m_configHelper.createConfigTypes( + EasyMock.capture(captureCluster), + EasyMock.capture(captureStackId), + EasyMock.capture(captureAmc), + EasyMock.capture(cap), + + EasyMock.capture(captureUsername), + EasyMock.capture(captureNote)); + expectLastCall().atLeastOnce(); + + replay(m_configHelper); + + RepositoryVersionEntity repoVersionEntity = helper.getOrCreateRepositoryVersion(new StackId("HDP-2.5.0"), "2.5.0-1234"); + + Map<String, Object> upgradeRequestMap = new HashMap<>(); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, repoVersionEntity.getId().toString()); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_HDP-250"); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); + + UpgradeContextFactory contextFactory = injector.getInstance(UpgradeContextFactory.class); + UpgradeContext context = contextFactory.create(cluster, upgradeRequestMap); + + UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class); + upgradeHelper.updateDesiredRepositoriesAndConfigs(context); + + assertNotNull(clusterEnvMap); + assertTrue(clusterEnvMap.containsKey("a")); + } + /** * @param cluster * @param direction http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index e798a4c..e92a619 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -59,6 +59,8 @@ import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -118,14 +120,14 @@ public class AmbariContextTest { private static final ConfigFactory configFactory = createNiceMock(ConfigFactory.class); private static final Service mockService1 = createStrictMock(Service.class); - private static final Collection<String> blueprintServices = new HashSet<String>(); - private static final Map<String, Service> clusterServices = new HashMap<String, Service>(); - private static final Map<Long, ConfigGroup> configGroups = new HashMap<Long, ConfigGroup>(); + private static final Collection<String> blueprintServices = new HashSet<>(); + private static final Map<String, Service> clusterServices = new HashMap<>(); + private static final Map<Long, ConfigGroup> configGroups = new HashMap<>(); private Configuration bpConfiguration = null; private Configuration group1Configuration = null; private static final Collection<String> group1Hosts = Arrays.asList(HOST1, HOST2); - private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = new Capture<Set<ConfigGroupRequest>>(); + private Capture<Set<ConfigGroupRequest>> configGroupRequestCapture = new Capture<>(); @Before public void setUp() throws Exception { @@ -156,26 +158,26 @@ public class AmbariContextTest { f.set(null, hostComponentResourceProvider); // bp configuration - Map<String, Map<String, String>> bpProperties = new HashMap<String, Map<String, String>>(); - Map<String, String> bpType1Props = new HashMap<String, String>(); + Map<String, Map<String, String>> bpProperties = new HashMap<>(); + Map<String, String> bpType1Props = new HashMap<>(); bpProperties.put("type1", bpType1Props); bpType1Props.put("prop1", "val1"); bpType1Props.put("prop2", "val2"); bpConfiguration = new Configuration(bpProperties, null); // host group 1 configuration - Map<String, Map<String, String>> group1Properties = new HashMap<String, Map<String, String>>(); - Map<String, String> type1Props = new HashMap<String, String>(); + Map<String, Map<String, String>> group1Properties = new HashMap<>(); + Map<String, String> type1Props = new HashMap<>(); group1Properties.put("type1", type1Props); type1Props.put("prop1", "val1.2"); type1Props.put("prop3", "val3"); group1Configuration = new Configuration(group1Properties, null, bpConfiguration); - Map<String, String> group1ResolvedProperties = new HashMap<String, String>(bpType1Props); + Map<String, String> group1ResolvedProperties = new HashMap<>(bpType1Props); group1ResolvedProperties.putAll(type1Props); // config type -> service mapping - Map<String, String> configTypeServiceMapping = new HashMap<String, String>(); + Map<String, String> configTypeServiceMapping = new HashMap<>(); configTypeServiceMapping.put("type1", "service1"); // config groups @@ -202,6 +204,14 @@ public class AmbariContextTest { type1Service1).anyTimes(); replay(type1Service1); + RepositoryVersionDAO repositoryVersionDAO = createStrictMock(RepositoryVersionDAO.class); + RepositoryVersionEntity repositoryVersion = createStrictMock(RepositoryVersionEntity.class); + expect(repositoryVersion.getId()).andReturn(1L).anyTimes(); + expect(repositoryVersionDAO.findByStackAndVersion(EasyMock.anyObject(StackId.class), + EasyMock.anyString())).andReturn(repositoryVersion).anyTimes(); + replay(repositoryVersionDAO, repositoryVersion); + context.repositoryVersionDAO = repositoryVersionDAO; + context.configFactory = configFactory; blueprintServices.add("service1"); @@ -270,23 +280,23 @@ public class AmbariContextTest { @Test public void testCreateAmbariResources() throws Exception { // expectations - Capture<ClusterRequest> clusterRequestCapture = new Capture<ClusterRequest>(); + Capture<ClusterRequest> clusterRequestCapture = new Capture<>(); controller.createCluster(capture(clusterRequestCapture)); expectLastCall().once(); expect(cluster.getServices()).andReturn(clusterServices).anyTimes(); - Capture<Set<ServiceRequest>> serviceRequestCapture = new Capture<Set<ServiceRequest>>(); - Capture<Set<ServiceComponentRequest>> serviceComponentRequestCapture = new Capture<Set<ServiceComponentRequest>>(); + Capture<Set<ServiceRequest>> serviceRequestCapture = new Capture<>(); + Capture<Set<ServiceComponentRequest>> serviceComponentRequestCapture = new Capture<>(); serviceResourceProvider.createServices(capture(serviceRequestCapture)); expectLastCall().once(); componentResourceProvider.createComponents(capture(serviceComponentRequestCapture)); expectLastCall().once(); - Capture<Request> serviceInstallRequestCapture = new Capture<Request>(); - Capture<Request> serviceStartRequestCapture = new Capture<Request>(); - Capture<Predicate> installPredicateCapture = new Capture<Predicate>(); - Capture<Predicate> startPredicateCapture = new Capture<Predicate>(); + Capture<Request> serviceInstallRequestCapture = new Capture<>(); + Capture<Request> serviceStartRequestCapture = new Capture<>(); + Capture<Predicate> installPredicateCapture = new Capture<>(); + Capture<Predicate> startPredicateCapture = new Capture<>(); expect(serviceResourceProvider.updateResources(capture(serviceInstallRequestCapture), capture(installPredicateCapture))).andReturn(null).once(); @@ -305,7 +315,7 @@ public class AmbariContextTest { Collection<ServiceRequest> serviceRequests = serviceRequestCapture.getValue(); assertEquals(2, serviceRequests.size()); - Collection<String> servicesFound = new HashSet<String>(); + Collection<String> servicesFound = new HashSet<>(); for (ServiceRequest serviceRequest : serviceRequests) { servicesFound.add(serviceRequest.getServiceName()); assertEquals(CLUSTER_NAME, serviceRequest.getClusterName()); @@ -315,13 +325,13 @@ public class AmbariContextTest { Collection<ServiceComponentRequest> serviceComponentRequests = serviceComponentRequestCapture.getValue(); assertEquals(3, serviceComponentRequests.size()); - Map<String, Collection<String>> foundServiceComponents = new HashMap<String, Collection<String>>(); + Map<String, Collection<String>> foundServiceComponents = new HashMap<>(); for (ServiceComponentRequest componentRequest : serviceComponentRequests) { assertEquals(CLUSTER_NAME, componentRequest.getClusterName()); String serviceName = componentRequest.getServiceName(); Collection<String> serviceComponents = foundServiceComponents.get(serviceName); if (serviceComponents == null) { - serviceComponents = new HashSet<String>(); + serviceComponents = new HashSet<>(); foundServiceComponents.put(serviceName, serviceComponents); } serviceComponents.add(componentRequest.getComponentName()); @@ -342,7 +352,7 @@ public class AmbariContextTest { Map<String, Object> installProperties = installPropertiesSet.iterator().next(); assertEquals(CLUSTER_NAME, installProperties.get(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID)); assertEquals("INSTALLED", installProperties.get(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID)); - assertEquals(new EqualsPredicate<String>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME), + assertEquals(new EqualsPredicate<>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME), installPredicateCapture.getValue()); Request startRequest = serviceStartRequestCapture.getValue(); @@ -351,7 +361,7 @@ public class AmbariContextTest { Map<String, Object> startProperties = startPropertiesSet.iterator().next(); assertEquals(CLUSTER_NAME, startProperties.get(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID)); assertEquals("STARTED", startProperties.get(ServiceResourceProvider.SERVICE_SERVICE_STATE_PROPERTY_ID)); - assertEquals(new EqualsPredicate<String>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME), + assertEquals(new EqualsPredicate<>(ServiceResourceProvider.SERVICE_CLUSTER_NAME_PROPERTY_ID, CLUSTER_NAME), installPredicateCapture.getValue()); } @@ -555,7 +565,7 @@ public class AmbariContextTest { testClusterEnvDesiredConfig.setTag(topologyResolvedState); Map<String, DesiredConfig> testDesiredConfigs = - new HashMap<String, DesiredConfig>(); + new HashMap<>(); testDesiredConfigs.put("hdfs-site", testHdfsDesiredConfig); testDesiredConfigs.put("core-site", testCoreSiteDesiredConfig); testDesiredConfigs.put("cluster-env", testClusterEnvDesiredConfig); @@ -565,7 +575,7 @@ public class AmbariContextTest { replayAll(); Set<String> testUpdatedConfigTypes = - new HashSet<String>(); + new HashSet<>(); testUpdatedConfigTypes.add("hdfs-site"); testUpdatedConfigTypes.add("core-site"); testUpdatedConfigTypes.add("cluster-env"); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java index fbf5e05..d082938 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java @@ -113,7 +113,7 @@ public class UpgradeCatalogTest { Set<UpgradeCatalog> upgradeCatalogSet = schemaUpgradeHelper.getAllUpgradeCatalogs(); Assert.assertNotNull(upgradeCatalogSet); - Assert.assertEquals(4, upgradeCatalogSet.size()); + Assert.assertEquals(3, upgradeCatalogSet.size()); List<UpgradeCatalog> upgradeCatalogs = schemaUpgradeHelper.getUpgradePath(null, "2.5.1"); http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/python/custom_actions/test_ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py index 8e03b7f..8c36ad6 100644 --- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py +++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py @@ -243,19 +243,7 @@ class TestRUSetAll(RMFTestCase): ru_execute.unlink_all_configs(None) # verify that os.path.islink was called for each conf - self.assertTrue(islink_mock.called) - for key, value in conf_select.get_package_dirs().iteritems(): - for directory_mapping in value: - original_config_directory = directory_mapping['conf_dir'] - is_link_called = False - - for call in islink_mock.call_args_list: - call_tuple = call[0] - if original_config_directory in call_tuple: - is_link_called = True - - if not is_link_called: - self.fail("os.path.islink({0}) was never called".format(original_config_directory)) + self.assertFalse(islink_mock.called) # alter JSON for a downgrade from 2.3 to 2.3 with open(json_file_path, "r") as json_file: @@ -313,37 +301,6 @@ class TestRUSetAll(RMFTestCase): # ensure it wasn't called this time self.assertFalse(islink_mock.called) - @patch("os.path.isdir") - @patch("os.path.islink") - def test_unlink_configs_missing_backup(self, islink_mock, isdir_mock): - - # required for the test to run since the Execute calls need this - from resource_management.core.environment import Environment - env = Environment(test_mode=True) - with env: - # Case: missing backup directory - isdir_mock.return_value = False - ru_execute = UpgradeSetAll() - self.assertEqual(len(env.resource_list), 0) - # Case: missing symlink - isdir_mock.reset_mock() - isdir_mock.return_value = True - islink_mock.return_value = False - ru_execute._unlink_config("/fake/config") - self.assertEqual(len(env.resource_list), 2) - # Case: missing symlink - isdir_mock.reset_mock() - isdir_mock.return_value = True - islink_mock.reset_mock() - islink_mock.return_value = True - - ru_execute._unlink_config("/fake/config") - self.assertEqual(pprint.pformat(env.resource_list), - "[Directory['/fake/config'],\n " - "Execute[('mv', '/fake/conf.backup', '/fake/config')],\n " - "Execute[('rm', '/fake/config')],\n " - "Execute[('mv', '/fake/conf.backup', '/fake/config')]]") - @patch("os.path.exists") @patch("os.path.islink") @patch("os.path.isdir") @@ -416,4 +373,4 @@ class TestRUSetAll(RMFTestCase): self.assertResourceCalled('Link', '/old/config', to = '/link/config', ) - self.assertNoMoreResources() \ No newline at end of file + self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml new file mode 100644 index 0000000..3e61696 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml @@ -0,0 +1,42 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-pack.xsd"> + <target>2.2.*.*</target> + <target-stack>HDP-2.2.0</target-stack> + <type>ROLLING</type> + <prerequisite-checks/> + + <order> + <group xsi:type="cluster" name="CLUSTER_SERVER_ACTIONS" title="Cluster Server Actions"> + <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Execute Foo/Bar Function"> + <task xsi:type="execute"> + <script>scripts/foo.py</script> + <function>bar</function> + </task> + </execute-stage> + </group> + </order> + + <processing> + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <upgrade /> + </component> + </service> + </processing> +</upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/abb9b463/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml new file mode 100644 index 0000000..6b08f27 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml @@ -0,0 +1,267 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-pack.xsd"> + <target>2.2.*.*</target> + <target-stack>HDP-2.5.0</target-stack> + <type>ROLLING</type> + <prerequisite-checks> + <!-- List of additional pre-req checks to run in addition to the required pre-reqs --> + <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check> + <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check> + <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check> + <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check> + <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check> + </prerequisite-checks> + + <order> + <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}"> + <execute-stage title="Confirm 1"> + <task xsi:type="manual"> + <message>Foo</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HIVE"> + <task xsi:type="manual"> + <message>Back stuff up.</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS"> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </execute-stage> + <execute-stage title="Confirm 2"> + <task xsi:type="manual"> + <message>Foo</message> + </task> + </execute-stage> + </group> + + <group name="ZOOKEEPER" title="Zookeeper"> + <skippable>true</skippable> + <allow-retry>false</allow-retry> + <service name="ZOOKEEPER"> + <component>ZOOKEEPER_SERVER</component> + <component>ZOOKEEPER_CLIENT</component> + </service> + </group> + + <group name="CORE_MASTER" title="Core Masters"> + <service name="HDFS"> + <component>JOURNALNODE</component> + <component>NAMENODE</component> + </service> + <service name="YARN"> + <component>RESOURCEMANAGER</component> + </service> + </group> + + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> + <skippable>true</skippable> <!-- set skippable for test --> + <allow-retry>false</allow-retry> <!-- set no retry for test --> + <service name="HDFS"> + <component>DATANODE</component> + </service> + <service name="HBASE"> + <component>REGIONSERVER</component> + </service> + <service name="YARN"> + <component>NODEMANAGER</component> + </service> + + <batch> + <percent>20</percent> + <message>Please run additional tests on {{components}}</message> + </batch> + </group> + + <group name="HIVE" title="Hive"> + <skippable>true</skippable> + <service name="HIVE"> + <component>HIVE_METASTORE</component> + <component>HIVE_SERVER</component> + <component>WEBHCAT_SERVER</component> + </service> + </group> + + <group name="OOZIE" title="Oozie"> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <service-check>false</service-check> + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + <component>OOZIE_CLIENT</component> + </service> + </group> + + <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> + <execute-stage title="Confirm Finalize"> + <task xsi:type="manual"> + <message>Please confirm you are ready to finalize</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </execute-stage> + <execute-stage title="Save Cluster State"> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> + </task> + </execute-stage> + </group> + + </order> + + + <processing> + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <summary>SUMMARY OF PREPARE</summary> + <message>This is a manual task with a placeholder of {{foo/bar}}</message> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade> + <task xsi:type="restart-task" timeout-config="upgrade.parameter.zk-server.timeout"/> + </upgrade> + <post-upgrade> + <task xsi:type="configure" id="hdp_2_1_1_zookeeper_new_config_type" /> + </post-upgrade> + <post-downgrade copy-upgrade="true" /> + </component> + </service> + + <service name="HDFS"> + <component name="NAMENODE"> + <pre-upgrade> + <task xsi:type="execute" hosts="master"> + <script>foo</script> + <function>list</function> + </task> + <task xsi:type="configure" id="hdp_2_1_1_nn_pre_upgrade" /> + <task xsi:type="manual"> + <message>{{direction.verb.proper}} your database</message> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-upgrade> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </post-upgrade> + <post-downgrade copy-upgrade="true" /> + </component> + <component name="DATANODE"> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-downgrade> + <task xsi:type="manual"> + <message>Manual Downgrade</message> + </task> + </post-downgrade> + </component> + </service> + + <service name="YARN"> + <component name="RESOURCEMANAGER"> + <pre-upgrade> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade /> + </component> + <component name="NODEMANAGER"> + <pre-upgrade> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + <task xsi:type="configure" id="hdp_2_1_1_nm_pre_upgrade"/> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade /> + </component> + </service> + + <service name="HIVE"> + <component name="HIVE_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <summary>HiveServer Port Availability</summary> + <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> + </task> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions"/> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions_skip"/> + <task xsi:type="configure" id="hdp_2_1_1_no_conditions_met"/> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade /> + </component> + </service> + + <service name="OOZIE"> + <component name="OOZIE_SERVER"> + <pre-upgrade> + <!-- This is important, do not remove it since UpgradeHelperTest.java : + testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts + that these tasks each run on their own stage. --> + <task xsi:type="execute" hosts="all" sequential="true"> + <summary>Shut down all Oozie servers</summary> + <script>scripts/oozie_server.py</script> + <function>stop</function> + </task> + + <task xsi:type="execute" hosts="any" sequential="true"> + <summary>Upgrading the Oozie database and creating a new sharelib</summary> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + + <component name="OOZIE_CLIENT"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + </processing> +</upgrade>
