AMBARI-21451 - Expected Values Like original_stack Are Missing On Downgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a6ac40bc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a6ac40bc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a6ac40bc Branch: refs/heads/branch-feature-AMBARI-21348 Commit: a6ac40bc07e4c0bec207d0786e4e870a6fe194f2 Parents: 08f48c1 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Wed Jul 12 13:30:16 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Wed Jul 12 14:43:15 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/stack_features.py | 28 +++-- .../actionmanager/ExecutionCommandWrapper.java | 3 +- .../ambari/server/agent/ExecutionCommand.java | 14 +-- .../controller/ActionExecutionContext.java | 30 +++--- .../controller/AmbariActionExecutionHelper.java | 8 +- .../ClusterStackVersionResourceProvider.java | 2 +- .../internal/UpgradeResourceProvider.java | 107 +++++++++++-------- .../upgrades/FinalizeUpgradeAction.java | 18 +--- .../upgrades/UpgradeUserKerberosDescriptor.java | 41 +++---- .../ambari/server/state/UpgradeContext.java | 31 +++--- .../SPARK/1.2.1/package/scripts/params.py | 11 +- .../SPARK/1.2.1/package/scripts/setup_spark.py | 6 +- .../1.2.1/package/scripts/spark_service.py | 6 +- .../ComponentVersionCheckActionTest.java | 1 - .../upgrades/UpgradeActionTest.java | 2 - .../UpgradeUserKerberosDescriptorTest.java | 19 ++-- .../src/test/python/TestStackFeature.py | 44 ++++++-- .../python/custom_actions/test_ru_set_all.py | 6 +- .../test/python/stacks/2.0.6/configs/nn_eu.json | 2 +- .../stacks/2.0.6/configs/nn_eu_standby.json | 2 +- .../2.1/configs/hive-metastore-upgrade.json | 2 +- .../python/stacks/2.2/configs/knox_upgrade.json | 2 +- 22 files changed, 203 insertions(+), 182 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 7811e26..2c66728 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -104,7 +104,10 @@ def get_stack_feature_version(config): # something like 2.4.0.0-1234; represents the version for the command # (or None if this is a cluster install and it hasn't been calculated yet) - version = default("/commandParams/version", None) + # this is always guaranteed to be the correct version for the command, even in + # upgrade and downgrade scenarios + command_version = default("/commandParams/version", None) + command_stack = default("/commandParams/target_stack", None) # something like 2.4.0.0-1234 # (or None if this is a cluster install and it hasn't been calculated yet) @@ -114,13 +117,13 @@ def get_stack_feature_version(config): upgrade_direction = default("/commandParams/upgrade_direction", None) # start out with the value that's right 99% of the time - version_for_stack_feature_checks = version if version is not None else stack_version + version_for_stack_feature_checks = command_version if command_version is not None else stack_version # if this is not an upgrade, then we take the simple path if upgrade_direction is None: Logger.info( - "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2} -> {3}".format( - stack_version, version, current_cluster_version, version_for_stack_feature_checks)) + "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}-> {4}".format( + stack_version, current_cluster_version, command_stack, command_version, version_for_stack_feature_checks)) return version_for_stack_feature_checks @@ -129,15 +132,12 @@ def get_stack_feature_version(config): is_stop_command = _is_stop_command(config) if not is_stop_command: Logger.info( - "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2}, upgrade_direction={3} -> {4}".format( - stack_version, version, current_cluster_version, upgrade_direction, + "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}, Upgrade Direction={4} -> {5}".format( + stack_version, current_cluster_version, command_stack, command_version, upgrade_direction, version_for_stack_feature_checks)) return version_for_stack_feature_checks - original_stack = default("/commandParams/original_stack", None) - target_stack = default("/commandParams/target_stack", None) - # something like 2.5.0.0-5678 (or None) downgrade_from_version = default("/commandParams/downgrade_from_version", None) @@ -153,15 +153,13 @@ def get_stack_feature_version(config): # UPGRADE if current_cluster_version is not None: version_for_stack_feature_checks = current_cluster_version - elif original_stack is not None: - version_for_stack_feature_checks = original_stack else: - version_for_stack_feature_checks = version if version is not None else stack_version + version_for_stack_feature_checks = command_version if command_version is not None else stack_version Logger.info( - "Stack Feature Version Info: stack_version={0}, version={1}, current_cluster_version={2}, upgrade_direction={3}, original_stack={4}, target_stack={5}, downgrade_from_version={6}, stop_command={7} -> {8}".format( - stack_version, version, current_cluster_version, upgrade_direction, original_stack, - target_stack, downgrade_from_version, is_stop_command, version_for_stack_feature_checks)) + "Stack Feature Version Info: Cluster Stack={0}, Cluster Current Version={1}, Command Stack={2}, Command Version={3}, Upgrade Direction={4}, stop_command={5} -> {6}".format( + stack_version, current_cluster_version, command_stack, command_version, upgrade_direction, + is_stop_command, version_for_stack_feature_checks)) return version_for_stack_feature_checks http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java index 28946e7..8875314 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java @@ -198,8 +198,9 @@ public class ExecutionCommandWrapper { Map<String,String> commandParams = executionCommand.getCommandParams(); + // set the version for the command if it's not already set ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion(); - if (null != effectiveClusterVersion) { + if (null != effectiveClusterVersion && !commandParams.containsKey(KeyNames.VERSION)) { commandParams.put(KeyNames.VERSION, effectiveClusterVersion.getRepositoryVersion().getVersion()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 32fb37b..7948d30 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -67,7 +67,7 @@ public class ExecutionCommand extends AgentCommand { private String role; @SerializedName("hostLevelParams") - private Map<String, String> hostLevelParams = new HashMap<String, String>(); + private Map<String, String> hostLevelParams = new HashMap<>(); @SerializedName("roleParams") private Map<String, String> roleParams = null; @@ -77,7 +77,7 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("clusterHostInfo") private Map<String, Set<String>> clusterHostInfo = - new HashMap<String, Set<String>>(); + new HashMap<>(); @SerializedName("configurations") private Map<String, Map<String, String>> configurations; @@ -92,7 +92,7 @@ public class ExecutionCommand extends AgentCommand { private boolean forceRefreshConfigTagsBeforeExecution = false; @SerializedName("commandParams") - private Map<String, String> commandParams = new HashMap<String, String>(); + private Map<String, String> commandParams = new HashMap<>(); @SerializedName("serviceName") private String serviceName; @@ -104,10 +104,10 @@ public class ExecutionCommand extends AgentCommand { private String componentName; @SerializedName("kerberosCommandParams") - private List<Map<String, String>> kerberosCommandParams = new ArrayList<Map<String, String>>(); + private List<Map<String, String>> kerberosCommandParams = new ArrayList<>(); @SerializedName("localComponents") - private Set<String> localComponents = new HashSet<String>(); + private Set<String> localComponents = new HashSet<>(); @SerializedName("availableServices") private Map<String, String> availableServices = new HashMap<>(); @@ -149,7 +149,7 @@ public class ExecutionCommand extends AgentCommand { } public Map<String, Map<String, String>> getConfigurationCredentials() { - return this.configurationCredentials; + return configurationCredentials; } public String getCommandId() { @@ -434,6 +434,8 @@ public class ExecutionCommand extends AgentCommand { String USER_GROUPS = "user_groups"; String NOT_MANAGED_HDFS_PATH_LIST = "not_managed_hdfs_path_list"; String VERSION = "version"; + String SOURCE_STACK = "source_stack"; + String TARGET_STACK = "target_stack"; String REFRESH_TOPOLOGY = "refresh_topology"; String HOST_SYS_PREPPED = "host_sys_prepped"; String MAX_DURATION_OF_RETRIES = "max_duration_for_retries"; http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java index c361094..af506f2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java @@ -25,7 +25,7 @@ import java.util.Map; import org.apache.ambari.server.actionmanager.TargetHostType; import org.apache.ambari.server.controller.internal.RequestOperationLevel; import org.apache.ambari.server.controller.internal.RequestResourceFilter; -import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; /** * The context required to create tasks and stages for a custom action @@ -42,7 +42,7 @@ public class ActionExecutionContext { private String expectedComponentName; private boolean hostsInMaintenanceModeExcluded = true; private boolean allowRetry = false; - private StackId stackId; + private RepositoryVersionEntity repositoryVersion; /** * {@code true} if slave/client component failures should be automatically @@ -171,27 +171,29 @@ public class ActionExecutionContext { } /** - * Gets the stack to use for generating stack-associated values for a command. - * In some cases the cluster's stack is not the correct one to use, such as - * when distributing a repository. + * Gets the stack/version to use for generating stack-associated values for a + * command. In some cases the cluster's stack is not the correct one to use, + * such as when distributing a repository. * - * @return the stackId the stack to use when generating stack-specific content - * for the command. + * @return the repository for the stack/version to use when generating + * stack-specific content for the command. + * + * @return */ - public StackId getStackId() { - return stackId; + public RepositoryVersionEntity getRepositoryVersion() { + return repositoryVersion; } /** - * Sets the stack to use for generating stack-associated values for a command. - * In some cases the cluster's stack is not the correct one to use, such as - * when distributing a repository. + * Sets the stack/version to use for generating stack-associated values for a + * command. In some cases the cluster's stack is not the correct one to use, + * such as when distributing a repository. * * @param stackId * the stackId to use for stack-based properties on the command. */ - public void setStackId(StackId stackId) { - this.stackId = stackId; + public void setRepositoryVersion(RepositoryVersionEntity repositoryVersion) { + this.repositoryVersion = repositoryVersion; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index f75fb41..0638910 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -553,10 +553,12 @@ public class AmbariActionExecutionHelper { // set the host level params if not already set by whoever is creating this command if (!hostLevelParams.containsKey(STACK_NAME) || !hostLevelParams.containsKey(STACK_VERSION)) { - // see if the action context has a stack ID set to use, otherwise use the + // see if the action context has a repository set to use for the command, otherwise use the // cluster's current stack ID - StackId stackId = actionContext.getStackId() != null ? actionContext.getStackId() - : cluster.getCurrentStackVersion(); + StackId stackId = cluster.getCurrentStackVersion(); + if (null != actionContext.getRepositoryVersion()) { + stackId = actionContext.getRepositoryVersion().getStackId(); + } hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index 9ea6083..633fe8c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -713,7 +713,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), INSTALL_PACKAGES_ACTION, Collections.singletonList(filter), params); - actionContext.setStackId(stackId); + actionContext.setRepositoryVersion(repoVersion); actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true))); return actionContext; http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 1130026..858b7cf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -883,9 +883,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // keep track of which stack to use when building commands - an express // upgrade switches the stack half way through while other types move it in // the beginning - StackId effectiveStackId = upgradeContext.getTargetStackId(); + RepositoryVersionEntity effectiveRepositoryVersion = upgradeContext.getTargetRepositoryVersion(); if(upgradeContext.getType() == UpgradeType.NON_ROLLING ) { - effectiveStackId = upgradeContext.getSourceStackId(); + effectiveRepositoryVersion = upgradeContext.getSourceRepositoryVersion(); } for (UpgradeGroupHolder group : groups) { @@ -895,7 +895,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider if (upgradeContext.getType() == UpgradeType.NON_ROLLING && UpdateStackGrouping.class.equals(group.groupClass)) { - effectiveStackId = upgradeContext.getTargetStackId(); + effectiveRepositoryVersion = upgradeContext.getTargetRepositoryVersion(); } List<UpgradeItemEntity> itemEntities = new ArrayList<>(); @@ -919,7 +919,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider itemEntities.add(itemEntity); injectVariables(configHelper, cluster, itemEntity); - makeServerSideStage(upgradeContext, req, effectiveStackId, itemEntity, + makeServerSideStage(upgradeContext, req, effectiveRepositoryVersion, itemEntity, (ServerSideActionTask) task, skippable, supportsAutoSkipOnFailure, allowRetry, pack, configUpgradePack); } @@ -934,7 +934,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider injectVariables(configHelper, cluster, itemEntity); // upgrade items match a stage - createStage(upgradeContext, req, effectiveStackId, itemEntity, wrapper, skippable, + createStage(upgradeContext, req, effectiveRepositoryVersion, itemEntity, wrapper, + skippable, supportsAutoSkipOnFailure, allowRetry); } } @@ -1275,10 +1276,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * the upgrade context (not {@code null}). * @param request * the request to add the new stage to (not {@code null}). - * @param effectiveStackId - * the stack ID to use when building the command. This will determine - * things like stack tools and version information added to the - * command (not {@code null}). + * @param effectiveRepositoryVersion + * the stack/version to use when building the command. This will + * determine things like stack tools and version information added to + * the command (not {@code null}). * @param entity * the upgrade entity to add the new items to (not {@code null}). * @param wrapper @@ -1292,25 +1293,27 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * @throws AmbariException */ private void createStage(UpgradeContext context, RequestStageContainer request, - StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable, - boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException { + RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity, + StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure, + boolean allowRetry) throws AmbariException { switch (wrapper.getType()) { case CONFIGURE: case START: case STOP: case RESTART: - makeCommandStage(context, request, effectiveStackId, entity, wrapper, skippable, + makeCommandStage(context, request, effectiveRepositoryVersion, entity, wrapper, skippable, supportsAutoSkipOnFailure, allowRetry); break; case RU_TASKS: - makeActionStage(context, request, effectiveStackId, entity, wrapper, skippable, + makeActionStage(context, request, effectiveRepositoryVersion, entity, wrapper, skippable, supportsAutoSkipOnFailure, allowRetry); break; case SERVICE_CHECK: - makeServiceCheckStage(context, request, effectiveStackId, entity, wrapper, skippable, + makeServiceCheckStage(context, request, effectiveRepositoryVersion, entity, wrapper, + skippable, supportsAutoSkipOnFailure, allowRetry); break; default: @@ -1341,9 +1344,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * the upgrade context. * @param request * the request object to add the stage to. - * @param effectiveStackId - * the stack ID to use when generating content for the command. On - * some upgrade types, this may change during the course of the + * @param effectiveRepositoryVersion + * the stack/version to use when generating content for the command. + * On some upgrade types, this may change during the course of the * upgrade orchestration. An express upgrade changes this after * stopping all services. * @param entity @@ -1360,8 +1363,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * @throws AmbariException */ private void makeActionStage(UpgradeContext context, RequestStageContainer request, - StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable, - boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException { + RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity, + StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure, + boolean allowRetry) throws AmbariException { if (0 == wrapper.getHosts().size()) { throw new AmbariException( @@ -1369,13 +1373,14 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } Cluster cluster = context.getCluster(); + StackId effectiveStackId = effectiveRepositoryVersion.getStackId(); // add each host to this stage RequestResourceFilter filter = new RequestResourceFilter("", "", new ArrayList<>(wrapper.getHosts())); LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks()); - Map<String, String> params = getNewParameterMap(request, context); + Map<String, String> params = getNewParameterMap(request, context, effectiveRepositoryVersion); params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks()); // Apply additional parameters to the command that come from the stage. @@ -1403,7 +1408,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), EXECUTE_TASK_ROLE, Collections.singletonList(filter), params); - actionContext.setStackId(effectiveStackId); + actionContext.setRepositoryVersion(effectiveRepositoryVersion); // hosts in maintenance mode are excluded from the upgrade actionContext.setMaintenanceModeHostExcluded(true); @@ -1449,10 +1454,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * Upgrade Context * @param request * Container for stage - * @param effectiveStackId - * the stack ID to use when building the command. This will determine - * things like stack tools and version information added to the - * command (not {@code null}). + * @param effectiveRepositoryVersion + * the stack/version to use when building the command. This will + * determine things like stack tools and version information added to + * the command (not {@code null}). * @param entity * Upgrade Item * @param wrapper @@ -1464,10 +1469,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * @throws AmbariException */ private void makeCommandStage(UpgradeContext context, RequestStageContainer request, - StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable, - boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException { + RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity, + StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure, + boolean allowRetry) throws AmbariException { Cluster cluster = context.getCluster(); + StackId effectiveStackId = effectiveRepositoryVersion.getStackId(); List<RequestResourceFilter> filters = new ArrayList<>(); @@ -1490,7 +1497,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider break; } - Map<String, String> commandParams = getNewParameterMap(request, context); + Map<String, String> commandParams = getNewParameterMap(request, context, + effectiveRepositoryVersion); // Apply additional parameters to the command that come from the stage. applyAdditionalParameters(wrapper, commandParams); @@ -1498,7 +1506,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), function, filters, commandParams); - actionContext.setStackId(effectiveStackId); + actionContext.setRepositoryVersion(effectiveRepositoryVersion); actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration)); actionContext.setRetryAllowed(allowRetry); @@ -1540,7 +1548,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } private void makeServiceCheckStage(UpgradeContext context, RequestStageContainer request, - StackId effectiveStackId, UpgradeItemEntity entity, StageWrapper wrapper, boolean skippable, + RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity, + StageWrapper wrapper, boolean skippable, boolean supportsAutoSkipOnFailure, boolean allowRetry) throws AmbariException { List<RequestResourceFilter> filters = new ArrayList<>(); @@ -1550,8 +1559,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } Cluster cluster = context.getCluster(); + StackId effectiveStackId = effectiveRepositoryVersion.getStackId(); - Map<String, String> commandParams = getNewParameterMap(request, context); + Map<String, String> commandParams = getNewParameterMap(request, context, + effectiveRepositoryVersion); // Apply additional parameters to the command that come from the stage. applyAdditionalParameters(wrapper, commandParams); @@ -1559,7 +1570,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), "SERVICE_CHECK", filters, commandParams); - actionContext.setStackId(effectiveStackId); + actionContext.setRepositoryVersion(effectiveRepositoryVersion); actionContext.setTimeout(wrapper.getMaxTimeout(s_configuration)); actionContext.setRetryAllowed(allowRetry); actionContext.setAutoSkipFailures(context.isServiceCheckFailureAutoSkipped()); @@ -1586,7 +1597,9 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider stage.setStageId(stageId); entity.setStageId(Long.valueOf(stageId)); - Map<String, String> requestParams = getNewParameterMap(request, context); + Map<String, String> requestParams = getNewParameterMap(request, context, + effectiveRepositoryVersion); + s_commandExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, requestParams, jsons); request.addStages(Collections.singletonList(stage)); @@ -1599,10 +1612,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * upgrade context * @param request * upgrade request - * @param effectiveStackId - * the stack ID to use when building the command. This will determine - * things like stack tools and version information added to the - * command (not {@code null}). + * @param effectiveRepositoryVersion + * the stack/version to use when building the command. This will + * determine things like stack tools and version information added to + * the command (not {@code null}). * @param entity * a single of upgrade * @param task @@ -1617,14 +1630,18 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * @throws AmbariException */ private void makeServerSideStage(UpgradeContext context, RequestStageContainer request, - StackId effectiveStackId, UpgradeItemEntity entity, ServerSideActionTask task, + RepositoryVersionEntity effectiveRepositoryVersion, UpgradeItemEntity entity, + ServerSideActionTask task, boolean skippable, boolean supportsAutoSkipOnFailure, boolean allowRetry, UpgradePack upgradePack, ConfigUpgradePack configUpgradePack) throws AmbariException { Cluster cluster = context.getCluster(); + StackId effectiveStackId = effectiveRepositoryVersion.getStackId(); + + Map<String, String> commandParams = getNewParameterMap(request, context, + effectiveRepositoryVersion); - Map<String, String> commandParams = getNewParameterMap(request, context); commandParams.put(UpgradeContext.COMMAND_PARAM_UPGRADE_PACK, upgradePack.getName()); // Notice that this does not apply any params because the input does not specify a stage. @@ -1702,7 +1719,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider Role.AMBARI_SERVER_ACTION.toString(), Collections.<RequestResourceFilter> emptyList(), commandParams); - actionContext.setStackId(effectiveStackId); + actionContext.setRepositoryVersion(effectiveRepositoryVersion); actionContext.setTimeout(Short.valueOf((short) -1)); actionContext.setRetryAllowed(allowRetry); actionContext.setAutoSkipFailures(context.isComponentFailureAutoSkipped()); @@ -1743,11 +1760,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * following properties are already set: * <ul> * <li>{@link UpgradeContext#COMMAND_PARAM_CLUSTER_NAME} + * <li>{@link UpgradeContext#COMMAND_PARAM_SOURCE_STACK} + * <li>{@link UpgradeContext#COMMAND_PARAM_TARGET_STACK} * <li>{@link UpgradeContext#COMMAND_PARAM_VERSION} * <li>{@link UpgradeContext#COMMAND_PARAM_DIRECTION} - * <li>{@link UpgradeContext#COMMAND_PARAM_ORIGINAL_STACK} - * <li>{@link UpgradeContext#COMMAND_PARAM_TARGET_STACK} - * <li>{@link UpgradeContext#COMMAND_DOWNGRADE_FROM_VERSION} * <li>{@link UpgradeContext#COMMAND_PARAM_UPGRADE_TYPE} * <li>{@link KeyNames#REFRESH_CONFIG_TAGS_BEFORE_EXECUTION} - necessary in * order to have the commands contain the correct configurations. Otherwise, @@ -1761,10 +1777,13 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider * @return the initialized parameter map. */ private Map<String, String> getNewParameterMap(RequestStageContainer requestStageContainer, - UpgradeContext context) { + UpgradeContext context, RepositoryVersionEntity effectiveRepositoryVersion) { Map<String, String> parameters = context.getInitializedCommandParameters(); + parameters.put(UpgradeContext.COMMAND_PARAM_REQUEST_ID, String.valueOf(requestStageContainer.getId())); + + parameters.put(UpgradeContext.COMMAND_PARAM_VERSION, effectiveRepositoryVersion.getVersion()); return parameters; } http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java index 0e6f0c4..d531460 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java @@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.events.StackUpgradeFinishEvent; import org.apache.ambari.server.events.publishers.VersionEventPublisher; @@ -73,26 +74,13 @@ public class FinalizeUpgradeAction extends AbstractServerAction { public static final String CLUSTER_NAME_KEY = "cluster_name"; public static final String UPGRADE_DIRECTION_KEY = "upgrade_direction"; - public static final String VERSION_KEY = "version"; + public static final String VERSION_KEY = KeyNames.VERSION; + public static final String TARGET_STACK_KEY = KeyNames.TARGET_STACK; public static final String REQUEST_ID = "request_id"; public static final String PREVIOUS_UPGRADE_NOT_COMPLETED_MSG = "It is possible that a previous upgrade was not finalized. " + "For this reason, Ambari will not remove any configs. Please ensure that all database records are correct."; /** - * The original "current" stack of the cluster before the upgrade started. - * This is the same regardless of whether the current direction is - * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}. - */ - public static final String ORIGINAL_STACK_KEY = "original_stack"; - - /** - * The target upgrade stack before the upgrade started. This is the same - * regardless of whether the current direction is {@link Direction#UPGRADE} or - * {@link Direction#DOWNGRADE}. - */ - public static final String TARGET_STACK_KEY = "target_stack"; - - /** * The Cluster that this ServerAction implementation is executing on */ @Inject http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java index 60d02a3..842da95 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptor.java @@ -17,11 +17,16 @@ */ package org.apache.ambari.server.serveraction.upgrades; -import com.google.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentMap; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.orm.dao.ArtifactDAO; import org.apache.ambari.server.orm.entities.ArtifactEntity; @@ -37,10 +42,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentMap; +import com.google.inject.Inject; /** * Update the user-defined Kerberos Descriptor to work with the current stack. @@ -56,25 +58,6 @@ public class UpgradeUserKerberosDescriptor extends AbstractServerAction { * @see Direction */ private static final String UPGRADE_DIRECTION_KEY = "upgrade_direction"; - - /** - * The original "current" stack of the cluster before the upgrade started. - * This is the same regardless of whether the current direction is - * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}. - * - * @see Direction - */ - private static final String ORIGINAL_STACK_KEY = "original_stack"; - - /** - * The target upgrade stack before the upgrade started. This is the same - * regardless of whether the current direction is {@link Direction#UPGRADE} or - * {@link Direction#DOWNGRADE}. - * - * @see Direction - */ - private static final String TARGET_STACK_KEY = "target_stack"; - private final static String KERBEROS_DESCRIPTOR_NAME = "kerberos_descriptor"; private final static String KERBEROS_DESCRIPTOR_BACKUP_NAME = "kerberos_descriptor_backup"; @@ -104,21 +87,21 @@ public class UpgradeUserKerberosDescriptor extends AbstractServerAction { HostRoleCommand hostRoleCommand = getHostRoleCommand(); String clusterName = hostRoleCommand.getExecutionCommandWrapper().getExecutionCommand().getClusterName(); Cluster cluster = clusters.getCluster(clusterName); - List<String> messages = new ArrayList<String>(); - List<String> errorMessages = new ArrayList<String>(); + List<String> messages = new ArrayList<>(); + List<String> errorMessages = new ArrayList<>(); if (cluster != null) { logMessage(messages, "Obtaining the user-defined Kerberos descriptor"); - TreeMap<String, String> foreignKeys = new TreeMap<String, String>(); + TreeMap<String, String> foreignKeys = new TreeMap<>(); foreignKeys.put("cluster", String.valueOf(cluster.getClusterId())); ArtifactEntity entity = artifactDAO.findByNameAndForeignKeys("kerberos_descriptor", foreignKeys); KerberosDescriptor userDescriptor = (entity == null) ? null : kerberosDescriptorFactory.createInstance(entity.getArtifactData()); if (userDescriptor != null) { - StackId originalStackId = getStackIdFromCommandParams(ORIGINAL_STACK_KEY); - StackId targetStackId = getStackIdFromCommandParams(TARGET_STACK_KEY); + StackId originalStackId = cluster.getCurrentStackVersion(); + StackId targetStackId = getStackIdFromCommandParams(KeyNames.TARGET_STACK); if (isDowngrade()) { restoreDescriptor(foreignKeys, messages, errorMessages); http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index b97dc80..93e6393 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -62,21 +62,15 @@ public class UpgradeContext { public static final String COMMAND_PARAM_UPGRADE_TYPE = "upgrade_type"; public static final String COMMAND_PARAM_TASKS = "tasks"; public static final String COMMAND_PARAM_STRUCT_OUT = "structured_out"; - public static final String COMMAND_DOWNGRADE_FROM_VERSION = "downgrade_from_version"; - /** - * The original "current" stack of the cluster before the upgrade started. - * This is the same regardless of whether the current direction is - * {@link Direction#UPGRADE} or {@link Direction#DOWNGRADE}. - */ - public static final String COMMAND_PARAM_ORIGINAL_STACK = "original_stack"; + public static final String COMMAND_PARAM_SOURCE_STACK = KeyNames.SOURCE_STACK; + public static final String COMMAND_PARAM_TARGET_STACK = KeyNames.TARGET_STACK; - /** - * The target upgrade stack before the upgrade started. This is the same - * regardless of whether the current direction is {@link Direction#UPGRADE} or - * {@link Direction#DOWNGRADE}. - */ - public static final String COMMAND_PARAM_TARGET_STACK = "target_stack"; + @Deprecated + @Experimental( + feature = ExperimentalFeature.STACK_UPGRADES_BETWEEN_VENDORS, + comment = "This isn't needed anymore, but many python classes still use it") + public static final String COMMAND_PARAM_DOWNGRADE_FROM_VERSION = "downgrade_from_version"; /** * The cluster that the upgrade is for. @@ -528,6 +522,7 @@ public class UpgradeContext { * <ul> * <li>{@link #COMMAND_PARAM_CLUSTER_NAME} * <li>{@link #COMMAND_PARAM_DIRECTION} + * <li>{@link #COMMAND_PARAM_DOWNGRADE_FROM_VERSION} * <li>{@link #COMMAND_PARAM_UPGRADE_TYPE} * <li>{@link KeyNames#REFRESH_CONFIG_TAGS_BEFORE_EXECUTION} - necessary in * order to have the commands contain the correct configurations. Otherwise, @@ -542,8 +537,16 @@ public class UpgradeContext { public Map<String, String> getInitializedCommandParameters() { Map<String, String> parameters = new HashMap<>(); + Direction direction = getDirection(); parameters.put(COMMAND_PARAM_CLUSTER_NAME, m_cluster.getClusterName()); - parameters.put(COMMAND_PARAM_DIRECTION, getDirection().name().toLowerCase()); + parameters.put(COMMAND_PARAM_DIRECTION, direction.name().toLowerCase()); + + parameters.put(COMMAND_PARAM_SOURCE_STACK, m_fromRepositoryVersion.getStackId().getStackId()); + parameters.put(COMMAND_PARAM_TARGET_STACK, m_toRepositoryVersion.getStackId().getStackId()); + + if (direction == Direction.DOWNGRADE) { + parameters.put(COMMAND_PARAM_DOWNGRADE_FROM_VERSION, m_fromRepositoryVersion.getVersion()); + } if (null != getType()) { // use the serialized attributes of the enum to convert it to a string, http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py index b54b565..5dbbaed 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py @@ -21,6 +21,7 @@ limitations under the License. import socket import status_params from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions import StackFeature from resource_management.libraries.functions import Direction from setup_spark import * @@ -57,10 +58,8 @@ upgrade_direction = default("/commandParams/upgrade_direction", None) java_home = config['hostLevelParams']['java_home'] stack_name = status_params.stack_name stack_root = Script.get_stack_root() -stack_version_unformatted = config['hostLevelParams']['stack_version'] -if upgrade_direction == Direction.DOWNGRADE: - stack_version_unformatted = config['commandParams']['original_stack'].split("-")[1] -stack_version_formatted = format_stack_version(stack_version_unformatted) + +version_for_stack_feature_checks = get_stack_feature_version(config) sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() @@ -71,7 +70,7 @@ spark_conf = '/etc/spark/conf' hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") -if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): +if check_stack_feature(StackFeature.ROLLING_UPGRADE, version_for_stack_feature_checks): hadoop_home = stack_select.get_hadoop_dir("home") spark_conf = format("{stack_root}/current/{component_directory}/conf") spark_log_dir = config['configurations']['spark-env']['spark_log_dir'] @@ -212,7 +211,7 @@ dfs_type = default("/commandParams/dfs_type", "") # livy is only supported from HDP 2.5 has_livyserver = False -if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted) and "livy-env" in config['configurations']: +if check_stack_feature(StackFeature.SPARK_LIVY, version_for_stack_feature_checks) and "livy-env" in config['configurations']: livy_component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "LIVY_SERVER") livy_conf = format("{stack_root}/current/{livy_component_directory}/conf") livy_log_dir = config['configurations']['livy-env']['livy_log_dir'] http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py index 4034532..bc86a67 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py @@ -114,11 +114,11 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): mode=0644 ) - effective_version = params.version if upgrade_type is not None else params.stack_version_formatted + effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks if effective_version: effective_version = format_stack_version(effective_version) - if effective_version and check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version): + if check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version): File(os.path.join(params.spark_conf, 'java-opts'), owner=params.spark_user, group=params.spark_group, @@ -130,7 +130,7 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): action="delete" ) - if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): + if params.spark_thrift_fairscheduler_content and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"), owner=params.spark_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py index 31a296a..2838186 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/spark_service.py @@ -34,11 +34,11 @@ def spark_service(name, upgrade_type=None, action=None): if action == 'start': - effective_version = params.version if upgrade_type is not None else params.stack_version_formatted + effective_version = params.version if upgrade_type is not None else params.version_for_stack_feature_checks if effective_version: effective_version = format_stack_version(effective_version) - if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): + if name == 'jobhistoryserver' and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # copy spark-hdp-assembly.jar to hdfs copy_to_hdfs("spark", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) # create spark history directory @@ -58,7 +58,7 @@ def spark_service(name, upgrade_type=None, action=None): # Spark 1.3.1.2.3, and higher, which was included in HDP 2.3, does not have a dependency on Tez, so it does not # need to copy the tarball, otherwise, copy it. - if params.stack_version_formatted and check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.stack_version_formatted): + if check_stack_feature(StackFeature.TEZ_FOR_SPARK, params.version_for_stack_feature_checks): resource_created = copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) if resource_created: params.HdfsResource(None, action="execute") http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java index 47d2a81..6675c58 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java @@ -409,7 +409,6 @@ public class ComponentVersionCheckActionTest { Map<String, String> commandParams = new HashMap<>(); commandParams.put(FinalizeUpgradeAction.UPGRADE_DIRECTION_KEY, "upgrade"); commandParams.put(FinalizeUpgradeAction.VERSION_KEY, targetRepo); - commandParams.put(FinalizeUpgradeAction.ORIGINAL_STACK_KEY, sourceStack.getStackId()); commandParams.put(FinalizeUpgradeAction.TARGET_STACK_KEY, targetStack.getStackId()); ExecutionCommand executionCommand = new ExecutionCommand(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java index c9c0dd0..037e47b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java @@ -677,7 +677,6 @@ public class UpgradeActionTest { Map<String, String> commandParams = new HashMap<>(); commandParams.put(FinalizeUpgradeAction.UPGRADE_DIRECTION_KEY, "upgrade"); commandParams.put(FinalizeUpgradeAction.VERSION_KEY, targetRepo); - commandParams.put(FinalizeUpgradeAction.ORIGINAL_STACK_KEY, sourceStack.getStackId()); commandParams.put(FinalizeUpgradeAction.TARGET_STACK_KEY, targetStack.getStackId()); ExecutionCommand executionCommand = new ExecutionCommand(); @@ -860,7 +859,6 @@ public class UpgradeActionTest { Map<String, String> commandParams = new HashMap<>(); commandParams.put(FinalizeUpgradeAction.UPGRADE_DIRECTION_KEY, "upgrade"); commandParams.put(FinalizeUpgradeAction.VERSION_KEY, targetRepo); - commandParams.put(FinalizeUpgradeAction.ORIGINAL_STACK_KEY, sourceStack.getStackId()); commandParams.put(FinalizeUpgradeAction.TARGET_STACK_KEY, targetStack.getStackId()); ExecutionCommand executionCommand = new ExecutionCommand(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java index 6b80623..0da9088 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeUserKerberosDescriptorTest.java @@ -25,6 +25,11 @@ import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.agent.ExecutionCommand; @@ -33,6 +38,7 @@ import org.apache.ambari.server.orm.dao.ArtifactDAO; import org.apache.ambari.server.orm.entities.ArtifactEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; import org.apache.ambari.server.state.kerberos.KerberosDescriptorUpdateHelper; @@ -45,11 +51,6 @@ import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - /** * Tests OozieConfigCalculation logic */ @@ -63,6 +64,7 @@ public class UpgradeUserKerberosDescriptorTest { private ArtifactDAO artifactDAO; private TreeMap<String, Field> fields = new TreeMap<>(); + private StackId HDP_24 = new StackId("HDP", "2.4"); @Before public void setup() throws Exception { @@ -74,6 +76,7 @@ public class UpgradeUserKerberosDescriptorTest { expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); expect(cluster.getClusterId()).andReturn(1l).atLeastOnce(); + expect(cluster.getCurrentStackVersion()).andReturn(HDP_24).atLeastOnce(); replay(clusters, cluster); prepareFields(); @@ -83,10 +86,9 @@ public class UpgradeUserKerberosDescriptorTest { @Test public void testUpgrade() throws Exception { - Map<String, String> commandParams = new HashMap<String, String>(); + Map<String, String> commandParams = new HashMap<>(); commandParams.put("clusterName", "c1"); commandParams.put("upgrade_direction", "UPGRADE"); - commandParams.put("original_stack", "HDP-2.4"); commandParams.put("target_stack", "HDP-2.5"); ExecutionCommand executionCommand = new ExecutionCommand(); @@ -141,10 +143,9 @@ public class UpgradeUserKerberosDescriptorTest { @Test public void testDowngrade() throws Exception { - Map<String, String> commandParams = new HashMap<String, String>(); + Map<String, String> commandParams = new HashMap<>(); commandParams.put("clusterName", "c1"); commandParams.put("upgrade_direction", "DOWNGRADE"); - commandParams.put("original_stack", "HDP-2.4"); commandParams.put("target_stack", "HDP-2.5"); ExecutionCommand executionCommand = new ExecutionCommand(); http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/TestStackFeature.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestStackFeature.py b/ambari-server/src/test/python/TestStackFeature.py index 0116a7a..230734c 100644 --- a/ambari-server/src/test/python/TestStackFeature.py +++ b/ambari-server/src/test/python/TestStackFeature.py @@ -28,6 +28,32 @@ from unittest import TestCase Logger.initialize_logger() class TestStackFeature(TestCase): + """ + EU Upgrade (HDP 2.5 to HDP 2.6) + - STOP + hostLevelParams/stack_name = HDP + hostLevelParams/stack_version = 2.5 + hostLevelParams/current_version = 2.5.0.0-1237 + commandParams/version = 2.5.0.0-1237 + - START + hostLevelParams/stack_name = HDP + hostLevelParams/stack_version = 2.6 + hostLevelParams/current_version = 2.5.0.0-1237 + commandParams/version = 2.6.0.0-334 + + EU Downgrade (HDP 2.6 to HDP 2.5) + - STOP + hostLevelParams/stack_name = HDP + hostLevelParams/stack_version = 2.6 + hostLevelParams/current_version = 2.5.0.0-1237 + commandParams/version = 2.6.0.0-334 + - START + hostLevelParams/stack_name = HDP + hostLevelParams/stack_version = 2.5 + hostLevelParams/current_version = 2.5.0.0-1237 + commandParams/version = 2.5.0.0-1237 + """ + def test_get_stack_feature_version_missing_params(self): try: stack_feature_version = get_stack_feature_version({}) @@ -122,7 +148,7 @@ class TestStackFeature(TestCase): "current_version": "2.4.0.0-1234" }, "commandParams": { - "original_stack": "2.4", + "source_stack": "2.4", "target_stack": "2.5", "upgrade_direction": "upgrade", "version": "2.5.9.9-9999" @@ -143,8 +169,8 @@ class TestStackFeature(TestCase): "current_version":"2.4.0.0-1234" }, "commandParams":{ - "original_stack":"2.4", - "target_stack":"2.5", + "source_stack":"2.5", + "target_stack":"2.4", "upgrade_direction":"downgrade", "version":"2.4.0.0-1234", "downgrade_from_version": "2.5.9.9-9999" @@ -166,10 +192,10 @@ class TestStackFeature(TestCase): "current_version":"2.4.0.0-1234" }, "commandParams":{ - "original_stack":"2.4", - "target_stack":"2.5", + "source_stack":"2.5", + "target_stack":"2.4", "upgrade_direction":"downgrade", - "version":"2.4.0.0-1234", + "version":"2.5.9.9-9999", "downgrade_from_version":"2.5.9.9-9999" } } @@ -189,10 +215,10 @@ class TestStackFeature(TestCase): "custom_command":"STOP" }, "commandParams":{ - "original_stack":"2.4", - "target_stack":"2.5", + "source_stack":"2.5", + "target_stack":"2.4", "upgrade_direction":"downgrade", - "version":"2.4.0.0-1234", + "version":"2.5.9.9-9999", "downgrade_from_version":"2.5.9.9-9999" } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/custom_actions/test_ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py index e1a89a8..8e03b7f 100644 --- a/ambari-server/src/test/python/custom_actions/test_ru_set_all.py +++ b/ambari-server/src/test/python/custom_actions/test_ru_set_all.py @@ -225,7 +225,7 @@ class TestRUSetAll(RMFTestCase): # alter JSON for a downgrade from 2.3 to 2.2 json_payload['commandParams']['version'] = "2.2.0.0-1234" json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-1234" - json_payload['commandParams']['original_stack'] = "HDP-2.2" + json_payload['commandParams']['source_stack'] = "HDP-2.2" json_payload['commandParams']['target_stack'] = "HDP-2.3" json_payload['commandParams']['upgrade_direction'] = "downgrade" json_payload['hostLevelParams']['stack_version'] = "2.2" @@ -263,7 +263,7 @@ class TestRUSetAll(RMFTestCase): json_payload['commandParams']['version'] = "2.3.0.0-1234" json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-5678" - json_payload['commandParams']['original_stack'] = "HDP-2.3" + json_payload['commandParams']['source_stack'] = "HDP-2.3" json_payload['commandParams']['target_stack'] = "HDP-2.3" json_payload['commandParams']['upgrade_direction'] = "downgrade" json_payload['hostLevelParams']['stack_version'] = "2.3" @@ -291,7 +291,7 @@ class TestRUSetAll(RMFTestCase): # alter JSON for a downgrade from 2.2 to 2.2 json_payload['commandParams']['version'] = "2.2.0.0-1234" json_payload['commandParams']['downgrade_from_version'] = "2.2.0.0-5678" - json_payload['commandParams']['original_stack'] = "HDP-2.2" + json_payload['commandParams']['source_stack'] = "HDP-2.2" json_payload['commandParams']['target_stack'] = "HDP-2.2" json_payload['commandParams']['upgrade_direction'] = "downgrade" json_payload['hostLevelParams']['stack_version'] = "2.2" http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json index 7f77d83..3aadf2c 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu.json @@ -25,7 +25,7 @@ "upgrade_type": "nonrolling_upgrade", "version": "2.3.2.0-2844", "forceRefreshConfigTagsBeforeExecution": "*", - "original_stack": "HDP-2.2", + "source_stack": "HDP-2.2", "command_timeout": "1200", "target_stack": "HDP-2.3", "desired_namenode_role": "standby", http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json index 87b18af..2d48ff6 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/nn_eu_standby.json @@ -25,7 +25,7 @@ "upgrade_type": "nonrolling_upgrade", "version": "2.3.2.0-2844", "forceRefreshConfigTagsBeforeExecution": "*", - "original_stack": "HDP-2.2", + "source_stack": "HDP-2.2", "command_timeout": "1200", "target_stack": "HDP-2.3", "desired_namenode_role": "standby", http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json index 99fcba0..021695b 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json @@ -13,7 +13,7 @@ "upgrade_type": "nonrolling_upgrade", "version": "2.3.2.0-2950", "forceRefreshConfigTagsBeforeExecution": "*", - "original_stack": "HDP-2.3", + "source_stack": "HDP-2.3", "command_timeout": "1200", "target_stack": "HDP-2.3", "script_type": "PYTHON" http://git-wip-us.apache.org/repos/asf/ambari/blob/a6ac40bc/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json index a9db11c..1805c3b 100644 --- a/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json +++ b/ambari-server/src/test/python/stacks/2.2/configs/knox_upgrade.json @@ -22,7 +22,7 @@ "upgrade_type": "rolling_upgrade", "command_retry_max_attempt_count": "3", "version": "2.3.0.0-2096", - "original_stack": "HDP-2.3", + "source_stack": "HDP-2.3", "command_retry_enabled": "false", "command_timeout": "1200", "target_stack": "HDP-2.3",