AMBARI-16687 - Configuration Tasks Are Being Skipped During Upgrade (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1209a706 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1209a706 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1209a706 Branch: refs/heads/branch-2.4 Commit: 1209a706dda1646bfeabc8059eb460b56aae1d6f Parents: ac151fb Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Mon May 16 14:48:10 2016 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Mon May 16 19:45:53 2016 -0400 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 89 +- .../server/stack/StackDefinitionDirectory.java | 2 +- .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 1 - .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 1284 +++++++++--------- .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 1 - .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 1 - .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 310 ++++- .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 1 - .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 1 - .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 166 +++ .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 10 - .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 186 --- .../HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml | 4 - .../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 2 - .../internal/UpgradeResourceProviderTest.java | 11 +- .../server/orm/InMemoryDefaultTestModule.java | 37 +- .../state/stack/ConfigUpgradeValidityTest.java | 192 +++ .../HDP/2.1.1/upgrades/config-upgrade.xml | 6 + .../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 2 +- .../HDP/2.1.1/upgrades/upgrade_test_partial.xml | 2 +- 20 files changed, 1358 insertions(+), 950 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1209a706/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index b069862..fb3ae69 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -815,7 +815,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider if (pack.getType() == UpgradeType.NON_ROLLING) { boolean foundGroupWithNameUPDATE_DESIRED_STACK_ID = false; for (UpgradeGroupHolder group : groups) { - if (group.name.equalsIgnoreCase(this.CONST_UPGRADE_GROUP_NAME)) { + if (group.name.equalsIgnoreCase(CONST_UPGRADE_GROUP_NAME)) { foundGroupWithNameUPDATE_DESIRED_STACK_ID = true; break; } @@ -823,7 +823,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider if (foundGroupWithNameUPDATE_DESIRED_STACK_ID == false) { throw new AmbariException(String.format("NonRolling Upgrade Pack %s requires a Group with name %s", - pack.getName(), this.CONST_UPGRADE_GROUP_NAME)); + pack.getName(), CONST_UPGRADE_GROUP_NAME)); } } @@ -846,27 +846,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider applyStackAndProcessConfigurations(targetStackId.getStackName(), cluster, version, direction, pack, userName); } - // Resolve or build a proper config upgrade pack - List<UpgradePack.IntermediateStack> intermediateStacks = pack.getIntermediateStacks(); - ConfigUpgradePack configUpgradePack; - - // No intermediate stacks - if (intermediateStacks == null || intermediateStacks.isEmpty()) { - configUpgradePack = s_metaProvider.get().getConfigUpgradePack( - targetStackId.getStackName(), targetStackId.getStackVersion()); - } else { - // For cross-stack upgrade, follow all major stacks and merge a new config upgrade pack from all - // target stacks involved into upgrade - ArrayList<ConfigUpgradePack> intermediateConfigUpgradePacks = new ArrayList<>(); - // Config change definitions are taken from all stacks up to (but excluding) target stack - intermediateStacks = intermediateStacks.subList(0, intermediateStacks.size() - 1); - for (UpgradePack.IntermediateStack intermediateStack : intermediateStacks) { - ConfigUpgradePack intermediateConfigUpgradePack = s_metaProvider.get().getConfigUpgradePack( - targetStackId.getStackName(), intermediateStack.version); - intermediateConfigUpgradePacks.add(intermediateConfigUpgradePack); - } - configUpgradePack = ConfigUpgradePack.merge(intermediateConfigUpgradePacks); - } + // resolve or build a proper config upgrade pack - always start out with the config pack + // for the current stack and merge into that + // + // HDP 2.2 to 2.3 should start with the config-upgrade.xml from HDP 2.2 + // HDP 2.2 to 2.4 should start with HDP 2.2 and merge in HDP 2.3's config-upgrade.xml + ConfigUpgradePack configUpgradePack = ConfigurationPackBuilder.build(pack, sourceStackId); // TODO: for now, all service components are transitioned to upgrading state // TODO: When performing patch upgrade, we should only target supported services/components @@ -1718,4 +1703,62 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } } } + + /** + * Builds the correct {@link ConfigUpgradePack} based on the upgrade and + * source stack. + * <ul> + * <li>HDP 2.2 to HDP 2.2 + * <ul> + * <li>Uses {@code config-upgrade.xml} from HDP 2.2 + * </ul> + * <li>HDP 2.2 to HDP 2.3 + * <ul> + * <li>Uses {@code config-upgrade.xml} from HDP 2.2 + * </ul> + * <li>HDP 2.2 to HDP 2.4 + * <ul> + * <li>Uses {@code config-upgrade.xml} from HDP 2.2 merged with the one from + * HDP 2.3 + * </ul> + * <ul> + */ + public static final class ConfigurationPackBuilder { + + /** + * Builds the configurations to use for the specified upgrade and source + * stack. + * + * @param upgradePack + * the upgrade pack (not {@code null}). + * @param sourceStackId + * the source stack (not {@code null}). + * @return the {@link ConfigUpgradePack} which contains all of the necessary + * configuration definitions for the upgrade. + */ + public static ConfigUpgradePack build(UpgradePack upgradePack, StackId sourceStackId) { + List<UpgradePack.IntermediateStack> intermediateStacks = upgradePack.getIntermediateStacks(); + ConfigUpgradePack configUpgradePack = s_metaProvider.get().getConfigUpgradePack( + sourceStackId.getStackName(), sourceStackId.getStackVersion()); + + // merge in any intermediate stacks + if (null != intermediateStacks) { + + // start out with the source stack's config pack + ArrayList<ConfigUpgradePack> configPacksToMerge = Lists.newArrayList(configUpgradePack); + + for (UpgradePack.IntermediateStack intermediateStack : intermediateStacks) { + ConfigUpgradePack intermediateConfigUpgradePack = s_metaProvider.get().getConfigUpgradePack( + sourceStackId.getStackName(), intermediateStack.version); + + configPacksToMerge.add(intermediateConfigUpgradePack); + } + + // merge all together + configUpgradePack = ConfigUpgradePack.merge(configPacksToMerge); + } + + return configUpgradePack; + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/1209a706/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java index f2e00fe..a385a3d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java @@ -37,7 +37,7 @@ public abstract class StackDefinitionDirectory { } }; - protected static final String CONFIG_UPGRADE_XML_FILENAME_PREFIX = "config-upgrade.xml"; + public static final String CONFIG_UPGRADE_XML_FILENAME_PREFIX = "config-upgrade.xml"; /** * underlying directory http://git-wip-us.apache.org/repos/asf/ambari/blob/1209a706/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml index 564b5bd..4c44afe 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml @@ -37,7 +37,6 @@ <!-- Instructs the upgrade pack how to build the configuration pack --> <upgrade-path> <intermediate-stack version="2.2"/> - <intermediate-stack version="2.3"/> </upgrade-path> <order>
