AMBARI-21617. Function ru_set_all is not working correctly due to changed command structure (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0a9f6fa9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0a9f6fa9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0a9f6fa9 Branch: refs/heads/feature-branch-AMBARI-21307 Commit: 0a9f6fa943e07164be0166981d0d65814068d34e Parents: c4a474a Author: Nate Cole <nc...@hortonworks.com> Authored: Mon Jul 31 14:31:37 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Wed Aug 2 07:57:22 2017 -0400 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 17 ++ .../ambari/server/state/ConfigHelper.java | 1 - .../ambari/server/state/UpgradeContext.java | 2 +- .../ambari/server/state/UpgradeHelper.java | 41 ++- .../custom_actions/scripts/ru_set_all.py | 31 --- .../ExecutionCommandWrapperTest.java | 70 ++++- .../server/api/services/AmbariMetaInfoTest.java | 2 + .../AmbariManagementControllerTest.java | 4 +- .../internal/UpgradeResourceProviderTest.java | 6 + .../ambari/server/state/UpgradeHelperTest.java | 81 +++++- .../server/topology/AmbariContextTest.java | 2 - .../python/custom_actions/test_ru_set_all.py | 33 +-- .../HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml | 267 +++++++++++++++++++ 13 files changed, 482 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index bc00f90..6b1fe05 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -99,6 +99,7 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper; import org.apache.ambari.server.state.stack.upgrade.Task; import org.apache.ambari.server.state.stack.upgrade.TaskWrapper; import org.apache.ambari.server.state.stack.upgrade.UpdateStackGrouping; +import org.apache.ambari.server.state.stack.upgrade.UpgradeScope; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent; import org.apache.ambari.server.utils.StageUtils; @@ -951,6 +952,22 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider Map<String, String> params = getNewParameterMap(request, context); params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks()); + // !!! when not scoped to a component (generic execution task) + if (context.isScoped(UpgradeScope.COMPLETE) && null == componentName) { + if (context.getDirection().isUpgrade()) { + params.put(KeyNames.VERSION, context.getRepositoryVersion().getVersion()); + } else { + // !!! in a full downgrade, the target version should be any of the history versions + UpgradeEntity lastUpgrade = s_upgradeDAO.findLastUpgradeForCluster( + cluster.getClusterId(), Direction.UPGRADE); + + @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES, + comment = "Shouldn't be getting the overall downgrade-to version.") + UpgradeHistoryEntity lastHistory = lastUpgrade.getHistory().iterator().next(); + params.put(KeyNames.VERSION, lastHistory.getFromReposistoryVersion().getVersion()); + } + } + // Apply additional parameters to the command that come from the stage. applyAdditionalParameters(wrapper, params); http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java index 359b225..5393f81 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java @@ -1118,7 +1118,6 @@ public class ConfigHelper { serviceMapped.put(service, new HashSet<Config>()); } serviceMapped.get(service).add(baseConfig); - } catch (Exception e) { // !!! ignore } http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index 8e7215c..59a1b02 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -675,7 +675,7 @@ public class UpgradeContext { * <p/> * If the direction is {@link Direction#UPGRADE} then this will return the * target repository which every service will be on if the upgrade is - * finalized. <br/> + * finalized. <p/> * If the direction is {@link Direction#DOWNGRADE} then this will return the * repository from which the downgrade is coming from. * http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index a5881d2..a5b40ff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; @@ -244,6 +245,8 @@ public class UpgradeHelper { if (StringUtils.isNotEmpty(preferredUpgradePackName) && packs.containsKey(preferredUpgradePackName)) { pack = packs.get(preferredUpgradePackName); + + LOG.warn("Upgrade pack '{}' not found for stack {}", preferredUpgradePackName, currentStack); } // Best-attempt at picking an upgrade pack assuming within the same stack whose target stack version matches. @@ -863,8 +866,11 @@ public class UpgradeHelper { String userName = controller.getAuthName(); Set<String> servicesInUpgrade = upgradeContext.getSupportedServices(); + Set<String> clusterConfigTypes = new HashSet<>(); + Set<String> processedClusterConfigTypes = new HashSet<>(); + // merge or revert configurations for any service that needs it - for( String serviceName : servicesInUpgrade ){ + for (String serviceName : servicesInUpgrade) { RepositoryVersionEntity sourceRepositoryVersion = upgradeContext.getSourceRepositoryVersion(serviceName); RepositoryVersionEntity targetRepositoryVersion = upgradeContext.getTargetRepositoryVersion(serviceName); StackId sourceStackId = sourceRepositoryVersion.getStackId(); @@ -901,6 +907,12 @@ public class UpgradeHelper { Map<String, Map<String, String>> newServiceDefaultConfigsByType = configHelper.getDefaultProperties( targetStackId, serviceName); + if (null == oldServiceDefaultConfigsByType || null == newServiceDefaultConfigsByType) { + continue; + } + + Set<String> foundConfigTypes = new HashSet<>(); + // find the current, existing configurations for the service List<Config> existingServiceConfigs = new ArrayList<>(); List<ServiceConfigEntity> latestServiceConfigs = m_serviceConfigDAO.getLastServiceConfigsForService( @@ -910,8 +922,23 @@ public class UpgradeHelper { List<ClusterConfigEntity> existingConfigurations = serviceConfig.getClusterConfigEntities(); for (ClusterConfigEntity currentServiceConfig : existingConfigurations) { String configurationType = currentServiceConfig.getType(); + Config currentClusterConfigForService = cluster.getDesiredConfigByType(configurationType); existingServiceConfigs.add(currentClusterConfigForService); + foundConfigTypes.add(configurationType); + } + } + + // !!! these are the types that come back from the config helper, but are not part of the service. + @SuppressWarnings("unchecked") + Set<String> missingConfigTypes = new HashSet<>(CollectionUtils.subtract(oldServiceDefaultConfigsByType.keySet(), + foundConfigTypes)); + + for (String missingConfigType : missingConfigTypes) { + Config config = cluster.getDesiredConfigByType(missingConfigType); + if (null != config) { + existingServiceConfigs.add(config); + clusterConfigTypes.add(missingConfigType); } } @@ -1013,8 +1040,18 @@ public class UpgradeHelper { } if (null != newServiceDefaultConfigsByType) { + + for (String clusterConfigType : clusterConfigTypes) { + if (processedClusterConfigTypes.contains(clusterConfigType)) { + newServiceDefaultConfigsByType.remove(clusterConfigType); + } else { + processedClusterConfigTypes.add(clusterConfigType); + } + + } + Set<String> configTypes = newServiceDefaultConfigsByType.keySet(); - LOG.info("The upgrade will create the following configurations for stack {}: {}", + LOG.warn("The upgrade will create the following configurations for stack {}: {}", targetStackId, StringUtils.join(configTypes, ',')); String serviceVersionNote = String.format("%s %s %s", direction.getText(true), http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py index 7b44677..95f7323 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py @@ -79,37 +79,6 @@ class UpgradeSetAll(Script): link_config(dir_def['conf_dir'], dir_def['current_dir']) - def _unlink_config(self, original_conf_directory): - """ - Reverses the work performed in link_config. This should only be used when downgrading from - HDP 2.3 to 2.2 in order to undo the conf symlink work required for 2.3. - - 1. Checks if conf.backup exists, if not then do no work - 2. Check for existance of 'etc' symlink and remove it - 3. Rename conf.back back to conf - - :original_conf_directory: the original conf directory that was made into a symlink (/etc/component/conf) - """ - # calculate the parent and backup directories - original_conf_parent_directory = os.path.abspath(os.path.join(original_conf_directory, os.pardir)) - backup_conf_directory = os.path.join(original_conf_parent_directory, "conf.backup") - Logger.info("Analyzing potential link {0}".format(original_conf_directory)) - - if os.path.islink(original_conf_directory): - # remove the old symlink - Execute(("rm", original_conf_directory), sudo=True) - elif os.path.isdir(original_conf_directory): - Directory(original_conf_directory, action="delete") - else: - Logger.info(" Skipping the unlink of {0}; it is not a symlink or does not exist".format(original_conf_directory)) - - if os.path.isdir(backup_conf_directory): - # rename the backup to the original name - Logger.info(" Unlinking {0} and restoring {1}".format(original_conf_directory, backup_conf_directory)) - Execute(("mv", backup_conf_directory, original_conf_directory), sudo=True) - else: - Logger.info(" Skipping restoring config from backup {0} since it does not exist".format(backup_conf_directory)) - def is_host_skippable(stack_selector_path, formatted_version): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java index 64a1e3a..52d54aa 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java @@ -18,7 +18,6 @@ package org.apache.ambari.server.actionmanager; - import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -34,6 +33,7 @@ import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.agent.AgentCommand.AgentCommandType; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.orm.OrmTestHelper; @@ -42,6 +42,7 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ConfigFactory; import org.apache.ambari.server.state.ConfigHelper; +import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent; import org.apache.ambari.server.utils.StageUtils; @@ -95,6 +96,7 @@ public class ExecutionCommandWrapperTest { private static ConfigFactory configFactory; private static ConfigHelper configHelper; private static StageFactory stageFactory; + private static OrmTestHelper ormTestHelper; @BeforeClass public static void setup() throws AmbariException { @@ -103,6 +105,7 @@ public class ExecutionCommandWrapperTest { configHelper = injector.getInstance(ConfigHelper.class); configFactory = injector.getInstance(ConfigFactory.class); stageFactory = injector.getInstance(StageFactory.class); + ormTestHelper = injector.getInstance(OrmTestHelper.class); clusters = injector.getInstance(Clusters.class); clusters.addHost(HOST1); @@ -273,6 +276,71 @@ public class ExecutionCommandWrapperTest { Assert.assertEquals(SERVICE_SITE_VAL6_H, mergedConfig.get(SERVICE_SITE_NAME6)); } + /** + * Test that the execution command wrapper properly sets the version + * information when the cluster is in the INSTALLING state. + * + * @throws JSONException + * @throws AmbariException + */ + @Test + public void testExecutionCommandHasVersionInfoWithoutCurrentClusterVersion() + throws JSONException, AmbariException { + Cluster cluster = clusters.getCluster(CLUSTER1); + + StackId stackId = cluster.getDesiredStackVersion(); + RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(stackId, "0.1-0000"); + Service service = cluster.getService("HDFS"); + service.setDesiredRepositoryVersion(repositoryVersion); + + // first try with an INSTALL command - this should not populate version info + ExecutionCommand executionCommand = new ExecutionCommand(); + Map<String, String> commandParams = new HashMap<>(); + + executionCommand.setClusterName(CLUSTER1); + executionCommand.setTaskId(1); + executionCommand.setRequestAndStage(1, 1); + executionCommand.setHostname(HOST1); + executionCommand.setRole("NAMENODE"); + executionCommand.setRoleParams(Collections.<String, String>emptyMap()); + executionCommand.setRoleCommand(RoleCommand.INSTALL); + executionCommand.setServiceName("HDFS"); + executionCommand.setCommandType(AgentCommandType.EXECUTION_COMMAND); + executionCommand.setCommandParams(commandParams); + + String json = StageUtils.getGson().toJson(executionCommand, ExecutionCommand.class); + ExecutionCommandWrapper execCommWrap = new ExecutionCommandWrapper(json); + injector.injectMembers(execCommWrap); + + ExecutionCommand processedExecutionCommand = execCommWrap.getExecutionCommand(); + commandParams = processedExecutionCommand.getCommandParams(); + Assert.assertTrue(commandParams.containsKey(KeyNames.VERSION)); + + // now try with a START command which should populate the version even + // though the state is INSTALLING + executionCommand = new ExecutionCommand(); + commandParams = new HashMap<>(); + + executionCommand.setClusterName(CLUSTER1); + executionCommand.setTaskId(1); + executionCommand.setRequestAndStage(1, 1); + executionCommand.setHostname(HOST1); + executionCommand.setRole("NAMENODE"); + executionCommand.setRoleParams(Collections.<String, String> emptyMap()); + executionCommand.setRoleCommand(RoleCommand.START); + executionCommand.setServiceName("HDFS"); + executionCommand.setCommandType(AgentCommandType.EXECUTION_COMMAND); + executionCommand.setCommandParams(commandParams); + + json = StageUtils.getGson().toJson(executionCommand, ExecutionCommand.class); + execCommWrap = new ExecutionCommandWrapper(json); + injector.injectMembers(execCommWrap); + + processedExecutionCommand = execCommWrap.getExecutionCommand(); + commandParams = processedExecutionCommand.getCommandParams(); + Assert.assertEquals("0.1-0000", commandParams.get(KeyNames.VERSION)); + } + @AfterClass public static void tearDown() throws AmbariException, SQLException { H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector); http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 4d2a8ae..5afe87e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -92,6 +92,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -226,6 +227,7 @@ public class AmbariMetaInfoTest { } @Test + @Ignore public void testGetRepositoryDefault() throws Exception { // Scenario: user has internet and does nothing to repos via api // use the latest http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 7a09d7b..5fb236b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -2253,7 +2253,7 @@ public class AmbariManagementControllerTest { resp = controller.getClusters(Collections.singleton(r)); Assert.assertTrue(resp.size() >= 3); - r = new ClusterRequest(null, null, "", null); + r = new ClusterRequest(null, null, null, null); resp = controller.getClusters(Collections.singleton(r)); Assert.assertTrue("Stack ID request is invalid and expect them all", resp.size() > 3); } @@ -6978,7 +6978,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(1, responsesWithParams.size()); StackVersionResponse resp = responsesWithParams.iterator().next(); assertNotNull(resp.getUpgradePacks()); - assertEquals(14, resp.getUpgradePacks().size()); + assertEquals(15, resp.getUpgradePacks().size()); assertTrue(resp.getUpgradePacks().contains("upgrade_test")); } http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java index 8c209ae..08c86dd 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java @@ -1601,6 +1601,12 @@ public class UpgradeResourceProviderTest extends EasyMockSupport { public void testCreatePatchRevertUpgrade() throws Exception { Cluster cluster = clusters.getCluster("c1"); + // add a single ZK server and client on 2.1.1.0 + Service service = cluster.addService("HBASE", repoVersionEntity2110); + ServiceComponent component = service.addServiceComponent("HBASE_MASTER"); + ServiceComponentHost sch = component.addServiceComponentHost("h1"); + sch.setVersion("2.1.1.0"); + File f = new File("src/test/resources/hbase_version_test.xml"); repoVersionEntity2112.setVersionXml(IOUtils.toString(new FileInputStream(f))); repoVersionEntity2112.setVersionXsd("version_definition.xsd"); http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index f0b884b..24a3fa2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -49,6 +49,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ClusterRequest; import org.apache.ambari.server.controller.ConfigurationRequest; +import org.apache.ambari.server.controller.internal.UpgradeResourceProvider; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.orm.OrmTestHelper; @@ -92,6 +93,7 @@ import org.junit.Before; import org.junit.Test; import org.springframework.security.core.context.SecurityContextHolder; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.gson.Gson; @@ -719,7 +721,6 @@ public class UpgradeHelperTest extends EasyMockSupport { List<ConfigUpgradeChangeDefinition.Transfer> transfers = m_gson.fromJson(configurationJson, new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); - System.out.println(">> transfers"+transfers); assertEquals(6, transfers.size()); assertEquals("copy-key", transfers.get(0).fromKey); @@ -797,8 +798,6 @@ public class UpgradeHelperTest extends EasyMockSupport { new TypeToken<List<ConfigUpgradeChangeDefinition.Transfer>>() { }.getType()); - System.out.println(" testConfigTaskConditionMet >> transfer"+transfers); - assertEquals("copy-key-one", transfers.get(0).fromKey); assertEquals("copy-to-key-one", transfers.get(0).toKey); @@ -2425,6 +2424,82 @@ public class UpgradeHelperTest extends EasyMockSupport { assertEquals("three-changed", expectedBazType.get("3")); } + @Test + public void testMergeConfigurationsWithClusterEnv() throws Exception { + Cluster cluster = makeCluster(true); + + StackId oldStack = cluster.getDesiredStackVersion(); + StackId newStack = new StackId("HDP-2.5.0"); + + ConfigFactory cf = injector.getInstance(ConfigFactory.class); + + Config clusterEnv = cf.createNew(cluster, "cluster-env", "version1", + ImmutableMap.<String, String>builder().put("a", "b").build(), + Collections.<String, Map<String, String>>emptyMap()); + + Config zooCfg = cf.createNew(cluster, "zoo.cfg", "version1", + ImmutableMap.<String, String>builder().put("c", "d").build(), + Collections.<String, Map<String, String>>emptyMap()); + + cluster.addDesiredConfig("admin", Sets.newHashSet(clusterEnv, zooCfg)); + + Map<String, Map<String, String>> stackMap = new HashMap<>(); + stackMap.put("cluster-env", new HashMap<String, String>()); + stackMap.put("hive-site", new HashMap<String, String>()); + + final Map<String, String> clusterEnvMap = new HashMap<>(); + + Capture<Cluster> captureCluster = Capture.newInstance(); + Capture<StackId> captureStackId = Capture.newInstance(); + Capture<AmbariManagementController> captureAmc = Capture.newInstance(); + + Capture<Map<String, Map<String, String>>> cap = new Capture<Map<String, Map<String, String>>>() { + @Override + public void setValue(Map<String, Map<String, String>> value) { + if (value.containsKey("cluster-env")) { + clusterEnvMap.putAll(value.get("cluster-env")); + } + } + }; + + Capture<String> captureUsername = Capture.newInstance(); + Capture<String> captureNote = Capture.newInstance(); + + EasyMock.reset(m_configHelper); + expect(m_configHelper.getDefaultProperties(oldStack, "HIVE")).andReturn(stackMap).atLeastOnce(); + expect(m_configHelper.getDefaultProperties(newStack, "HIVE")).andReturn(stackMap).atLeastOnce(); + expect(m_configHelper.getDefaultProperties(oldStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce(); + expect(m_configHelper.getDefaultProperties(newStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce(); + m_configHelper.createConfigTypes( + EasyMock.capture(captureCluster), + EasyMock.capture(captureStackId), + EasyMock.capture(captureAmc), + EasyMock.capture(cap), + + EasyMock.capture(captureUsername), + EasyMock.capture(captureNote)); + expectLastCall().atLeastOnce(); + + replay(m_configHelper); + + RepositoryVersionEntity repoVersionEntity = helper.getOrCreateRepositoryVersion(new StackId("HDP-2.5.0"), "2.5.0-1234"); + + Map<String, Object> upgradeRequestMap = new HashMap<>(); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, repoVersionEntity.getId().toString()); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_HDP-250"); + upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); + + UpgradeContextFactory contextFactory = injector.getInstance(UpgradeContextFactory.class); + UpgradeContext context = contextFactory.create(cluster, upgradeRequestMap); + + UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class); + upgradeHelper.updateDesiredRepositoriesAndConfigs(context); + + assertNotNull(clusterEnvMap); + assertTrue(clusterEnvMap.containsKey("a")); + } + /** * @param cluster * @param direction http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 9ec3621..13a14ac 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -120,7 +120,6 @@ public class AmbariContextTest { private static final ConfigFactory configFactory = createNiceMock(ConfigFactory.class); private static final Service mockService1 = createStrictMock(Service.class); - private static final Collection<String> blueprintServices = new HashSet<>(); private static final Map<String, Service> clusterServices = new HashMap<>(); private static final Map<Long, ConfigGroup> configGroups = new HashMap<>(); @@ -306,7 +305,6 @@ public class AmbariContextTest { capture(startPredicateCapture))).andReturn(null).once(); - replayAll(); // test http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/python/custom_actions/test_ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py index 29c99d8..a488a96 100644 --- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py +++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py @@ -195,37 +195,6 @@ class TestRUSetAll(RMFTestCase): self.assertEqual(call_mock.call_count, 1) - @patch("os.path.isdir") - @patch("os.path.islink") - def test_unlink_configs_missing_backup(self, islink_mock, isdir_mock): - - # required for the test to run since the Execute calls need this - from resource_management.core.environment import Environment - env = Environment(test_mode=True) - with env: - # Case: missing backup directory - isdir_mock.return_value = False - ru_execute = UpgradeSetAll() - self.assertEqual(len(env.resource_list), 0) - # Case: missing symlink - isdir_mock.reset_mock() - isdir_mock.return_value = True - islink_mock.return_value = False - ru_execute._unlink_config("/fake/config") - self.assertEqual(len(env.resource_list), 2) - # Case: missing symlink - isdir_mock.reset_mock() - isdir_mock.return_value = True - islink_mock.reset_mock() - islink_mock.return_value = True - - ru_execute._unlink_config("/fake/config") - self.assertEqual(pprint.pformat(env.resource_list), - "[Directory['/fake/config'],\n " - "Execute[('mv', '/fake/conf.backup', '/fake/config')],\n " - "Execute[('rm', '/fake/config')],\n " - "Execute[('mv', '/fake/conf.backup', '/fake/config')]]") - @patch("os.path.exists") @patch("os.path.islink") @patch("os.path.isdir") @@ -298,4 +267,4 @@ class TestRUSetAll(RMFTestCase): self.assertResourceCalled('Link', '/old/config', to = '/link/config', ) - self.assertNoMoreResources() \ No newline at end of file + self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/0a9f6fa9/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml new file mode 100644 index 0000000..6b08f27 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml @@ -0,0 +1,267 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:noNamespaceSchemaLocation="upgrade-pack.xsd"> + <target>2.2.*.*</target> + <target-stack>HDP-2.5.0</target-stack> + <type>ROLLING</type> + <prerequisite-checks> + <!-- List of additional pre-req checks to run in addition to the required pre-reqs --> + <check>org.apache.ambari.server.checks.HiveMultipleMetastoreCheck</check> + <check>org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck</check> + <check>org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck</check> + <check>org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck</check> + <check>org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck</check> + <check>org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck</check> + <check>org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck</check> + </prerequisite-checks> + + <order> + <group xsi:type="cluster" name="PRE_CLUSTER" title="Pre {{direction.text.proper}}"> + <execute-stage title="Confirm 1"> + <task xsi:type="manual"> + <message>Foo</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HIVE"> + <task xsi:type="manual"> + <message>Back stuff up.</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Finalize HDFS"> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </execute-stage> + <execute-stage title="Confirm 2"> + <task xsi:type="manual"> + <message>Foo</message> + </task> + </execute-stage> + </group> + + <group name="ZOOKEEPER" title="Zookeeper"> + <skippable>true</skippable> + <allow-retry>false</allow-retry> + <service name="ZOOKEEPER"> + <component>ZOOKEEPER_SERVER</component> + <component>ZOOKEEPER_CLIENT</component> + </service> + </group> + + <group name="CORE_MASTER" title="Core Masters"> + <service name="HDFS"> + <component>JOURNALNODE</component> + <component>NAMENODE</component> + </service> + <service name="YARN"> + <component>RESOURCEMANAGER</component> + </service> + </group> + + <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated"> + <skippable>true</skippable> <!-- set skippable for test --> + <allow-retry>false</allow-retry> <!-- set no retry for test --> + <service name="HDFS"> + <component>DATANODE</component> + </service> + <service name="HBASE"> + <component>REGIONSERVER</component> + </service> + <service name="YARN"> + <component>NODEMANAGER</component> + </service> + + <batch> + <percent>20</percent> + <message>Please run additional tests on {{components}}</message> + </batch> + </group> + + <group name="HIVE" title="Hive"> + <skippable>true</skippable> + <service name="HIVE"> + <component>HIVE_METASTORE</component> + <component>HIVE_SERVER</component> + <component>WEBHCAT_SERVER</component> + </service> + </group> + + <group name="OOZIE" title="Oozie"> + <skippable>true</skippable> + <supports-auto-skip-failure>false</supports-auto-skip-failure> + <service-check>false</service-check> + <service name="OOZIE"> + <component>OOZIE_SERVER</component> + <component>OOZIE_CLIENT</component> + </service> + </group> + + <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}"> + <execute-stage title="Confirm Finalize"> + <task xsi:type="manual"> + <message>Please confirm you are ready to finalize</message> + </task> + </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Execute HDFS Finalize"> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </execute-stage> + <execute-stage title="Save Cluster State"> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction"> + </task> + </execute-stage> + </group> + + </order> + + + <processing> + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <summary>SUMMARY OF PREPARE</summary> + <message>This is a manual task with a placeholder of {{foo/bar}}</message> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade> + <task xsi:type="restart-task" timeout-config="upgrade.parameter.zk-server.timeout"/> + </upgrade> + <post-upgrade> + <task xsi:type="configure" id="hdp_2_1_1_zookeeper_new_config_type" /> + </post-upgrade> + <post-downgrade copy-upgrade="true" /> + </component> + </service> + + <service name="HDFS"> + <component name="NAMENODE"> + <pre-upgrade> + <task xsi:type="execute" hosts="master"> + <script>foo</script> + <function>list</function> + </task> + <task xsi:type="configure" id="hdp_2_1_1_nn_pre_upgrade" /> + <task xsi:type="manual"> + <message>{{direction.verb.proper}} your database</message> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-upgrade> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </post-upgrade> + <post-downgrade copy-upgrade="true" /> + </component> + <component name="DATANODE"> + <pre-downgrade /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + <post-downgrade> + <task xsi:type="manual"> + <message>Manual Downgrade</message> + </task> + </post-downgrade> + </component> + </service> + + <service name="YARN"> + <component name="RESOURCEMANAGER"> + <pre-upgrade> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade /> + </component> + <component name="NODEMANAGER"> + <pre-upgrade> + <task xsi:type="execute"> + <script>foo</script> + <function>list</function> + </task> + <task xsi:type="configure" id="hdp_2_1_1_nm_pre_upgrade"/> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade /> + </component> + </service> + + <service name="HIVE"> + <component name="HIVE_SERVER"> + <pre-upgrade> + <task xsi:type="manual"> + <summary>HiveServer Port Availability</summary> + <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> + </task> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions"/> + <task xsi:type="configure" id="hdp_2_1_1_hive_server_conditions_skip"/> + <task xsi:type="configure" id="hdp_2_1_1_no_conditions_met"/> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade /> + </component> + </service> + + <service name="OOZIE"> + <component name="OOZIE_SERVER"> + <pre-upgrade> + <!-- This is important, do not remove it since UpgradeHelperTest.java : + testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts + that these tasks each run on their own stage. --> + <task xsi:type="execute" hosts="all" sequential="true"> + <summary>Shut down all Oozie servers</summary> + <script>scripts/oozie_server.py</script> + <function>stop</function> + </task> + + <task xsi:type="execute" hosts="any" sequential="true"> + <summary>Upgrading the Oozie database and creating a new sharelib</summary> + <script>scripts/oozie_server_upgrade.py</script> + <function>upgrade_oozie_database_and_sharelib</function> + </task> + </pre-upgrade> + <pre-downgrade copy-upgrade="true" /> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + + <component name="OOZIE_CLIENT"> + <upgrade> + <task xsi:type="restart-task" /> + </upgrade> + </component> + </service> + </processing> +</upgrade>
