AMBARI-12700. Stop-and-Start Upgrade: Move Configs out of Upgrade Pack (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a67ddd27 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a67ddd27 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a67ddd27 Branch: refs/heads/branch-dev-stop-all-upgrade Commit: a67ddd27d2f8e164d55df02b813904e64753772a Parents: 54146bb Author: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Authored: Wed Sep 9 18:32:40 2015 +0300 Committer: Lisnichenko Dmitro <dlysniche...@hortonworks.com> Committed: Wed Sep 9 18:32:40 2015 +0300 ---------------------------------------------------------------------- .../server/api/services/AmbariMetaInfo.java | 21 +- .../internal/UpgradeResourceProvider.java | 47 +- .../serveraction/upgrades/ConfigureAction.java | 23 +- .../server/stack/ModuleFileUnmarshaller.java | 4 +- .../server/stack/StackDefinitionDirectory.java | 2 + .../ambari/server/stack/StackDirectory.java | 53 +- .../apache/ambari/server/stack/StackModule.java | 2 +- .../apache/ambari/server/state/StackInfo.java | 29 +- .../ambari/server/state/UpgradeHelper.java | 15 +- .../server/state/stack/ConfigUpgradePack.java | 147 ++++ .../upgrade/ConfigUpgradeChangeDefinition.java | 420 ++++++++++ .../state/stack/upgrade/ConfigureTask.java | 328 ++------ .../server/state/stack/upgrade/Grouping.java | 2 +- .../stack/upgrade/StageWrapperBuilder.java | 2 +- .../stacks/HDP/2.2/upgrades/config-upgrade.xml | 55 ++ .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 26 +- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 621 ++------------ .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 805 +++++++++++++++++++ .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 26 +- .../upgrades/ConfigureActionTest.java | 36 +- .../ambari/server/state/UpgradeHelperTest.java | 395 ++++----- .../server/state/stack/UpgradePackTest.java | 212 ++--- 22 files changed, 2005 insertions(+), 1266 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 4afa9b0..1e74dfb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -60,6 +60,7 @@ import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptorFactory; import org.apache.ambari.server.state.stack.Metric; import org.apache.ambari.server.state.stack.MetricDefinition; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.UpgradePack; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -994,7 +995,7 @@ public class AmbariMetaInfo { } return alertDefinitionFactory.getAlertDefinitions(alertsFile, - service.getName()); + service.getName()); } /** @@ -1203,6 +1204,24 @@ public class AmbariMetaInfo { } /** + * Get all upgrade config pack if it is available for a stack. + * + * @param stackName the stack name + * @param stackVersion the stack version + * @return config upgrade pack for stack or null if it is + * not defined for stack + */ + public ConfigUpgradePack getConfigUpgradePack(String stackName, String stackVersion) { + try { + StackInfo stack = getStack(stackName, stackVersion); + return stack.getConfigUpgradePack(); + } catch (AmbariException e) { + LOG.debug("Cannot load config upgrade pack for non-existent stack {}-{}", stackName, stackVersion, e); + return null; + } + } + + /** * Gets the fully compiled Kerberos descriptor for the relevant stack and version. * <p/> * All of the kerberos.json files from the specified stack (and version) are read, parsed and http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index dddec73..2c9714e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -85,6 +85,7 @@ import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.UpgradeHelper; import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; import org.apache.ambari.server.state.stack.upgrade.Direction; @@ -488,10 +489,10 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } Map<String, UpgradePack> packs = s_metaProvider.get().getUpgradePacks(stack.getStackName(), - stack.getStackVersion()); + stack.getStackVersion()); UpgradePack pack = null; - if (!preferredUpgradePackName.isEmpty() && packs.containsKey(preferredUpgradePackName)) { + if (preferredUpgradePackName != null && !preferredUpgradePackName.isEmpty() && packs.containsKey(preferredUpgradePackName)) { pack = packs.get(preferredUpgradePackName); } @@ -617,6 +618,11 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider // names are read and set on the command for filling in later processConfigurations(targetStackId.getStackName(), cluster, version, direction, pack); + // TODO: for cross-stack upgrade, merge a new config upgrade pack from all + // target stacks involved into upgrade and pass it into method + ConfigUpgradePack configUpgradePack = s_metaProvider.get().getConfigUpgradePack( + targetStackId.getStackName(), targetStackId.getStackVersion()); + for (UpgradeGroupHolder group : groups) { UpgradeGroupEntity groupEntity = new UpgradeGroupEntity(); groupEntity.setName(group.name); @@ -642,7 +648,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider injectVariables(configHelper, cluster, itemEntity); makeServerSideStage(ctx, req, itemEntity, (ServerSideActionTask) task, skippable, - allowRetry); + allowRetry, configUpgradePack); } } } else { @@ -667,7 +673,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider entity.setFromVersion(cluster.getCurrentClusterVersion().getRepositoryVersion().getVersion()); entity.setToVersion(version); entity.setUpgradeGroups(groupEntities); - entity.setClusterId(Long.valueOf(cluster.getClusterId())); + entity.setClusterId(cluster.getClusterId()); entity.setDirection(direction); entity.setUpgradePackage(pack.getName()); entity.setUpgradeType(pack.getType()); @@ -1132,8 +1138,22 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider request.addStages(Collections.singletonList(stage)); } + /** + * Creates a stage consisting of server side actions + * @param context upgrade context + * @param request upgrade request + * @param entity a single of upgrade + * @param task server-side task (if any) + * @param skippable if user can skip stage on failure + * @param allowRetry if user can retry running stage on failure + * @param configUpgradePack a runtime-generated config upgrade pack that + * contains all config change definitions from all stacks involved into + * upgrade + * @throws AmbariException + */ private void makeServerSideStage(UpgradeContext context, RequestStageContainer request, - UpgradeItemEntity entity, ServerSideActionTask task, boolean skippable, boolean allowRetry) + UpgradeItemEntity entity, ServerSideActionTask task, boolean skippable, boolean allowRetry, + ConfigUpgradePack configUpgradePack) throws AmbariException { Cluster cluster = context.getCluster(); @@ -1166,7 +1186,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider } case CONFIGURE: { ConfigureTask ct = (ConfigureTask) task; - Map<String, String> configurationChanges = ct.getConfigurationChanges(cluster); + Map<String, String> configurationChanges = + ct.getConfigurationChanges(cluster, configUpgradePack); // add all configuration changes to the command params commandParams.putAll(configurationChanges); @@ -1219,13 +1240,13 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider entity.setStageId(Long.valueOf(stageId)); stage.addServerActionCommand(task.getImplementationClass(), - getManagementController().getAuthName(), - Role.AMBARI_SERVER_ACTION, - RoleCommand.EXECUTE, - cluster.getClusterName(), - new ServiceComponentHostServerActionEvent(null, - System.currentTimeMillis()), - commandParams, itemDetail, null, Integer.valueOf(1200), allowRetry); + getManagementController().getAuthName(), + Role.AMBARI_SERVER_ACTION, + RoleCommand.EXECUTE, + cluster.getClusterName(), + new ServiceComponentHostServerActionEvent(null, + System.currentTimeMillis()), + commandParams, itemDetail, null, Integer.valueOf(1200), allowRetry); request.addStages(Collections.singletonList(stage)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java index c717582..ef21a2a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java @@ -46,7 +46,10 @@ import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; -import org.apache.ambari.server.state.stack.upgrade.ConfigureTask.ConfigurationKeyValue; +import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue; +import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Transfer; +import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Replace; +import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Masked; import org.apache.commons.lang.StringUtils; import com.google.gson.Gson; @@ -176,27 +179,27 @@ public class ConfigureAction extends AbstractServerAction { String configType = commandParameters.get(ConfigureTask.PARAMETER_CONFIG_TYPE); // extract transfers - List<ConfigureTask.ConfigurationKeyValue> keyValuePairs = Collections.emptyList(); + List<ConfigurationKeyValue> keyValuePairs = Collections.emptyList(); String keyValuePairJson = commandParameters.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); if (null != keyValuePairJson) { keyValuePairs = m_gson.fromJson( - keyValuePairJson, new TypeToken<List<ConfigureTask.ConfigurationKeyValue>>(){}.getType()); + keyValuePairJson, new TypeToken<List<ConfigurationKeyValue>>(){}.getType()); } // extract transfers - List<ConfigureTask.Transfer> transfers = Collections.emptyList(); + List<Transfer> transfers = Collections.emptyList(); String transferJson = commandParameters.get(ConfigureTask.PARAMETER_TRANSFERS); if (null != transferJson) { transfers = m_gson.fromJson( - transferJson, new TypeToken<List<ConfigureTask.Transfer>>(){}.getType()); + transferJson, new TypeToken<List<Transfer>>(){}.getType()); } // extract replacements - List<ConfigureTask.Replace> replacements = Collections.emptyList(); + List<Replace> replacements = Collections.emptyList(); String replaceJson = commandParameters.get(ConfigureTask.PARAMETER_REPLACEMENTS); if (null != replaceJson) { replacements = m_gson.fromJson( - replaceJson, new TypeToken<List<ConfigureTask.Replace>>(){}.getType()); + replaceJson, new TypeToken<List<Replace>>(){}.getType()); } // if there is nothing to do, then skip the task @@ -240,7 +243,7 @@ public class ConfigureAction extends AbstractServerAction { // !!! do transfers first before setting defined values StringBuilder outputBuffer = new StringBuilder(250); - for (ConfigureTask.Transfer transfer : transfers) { + for (Transfer transfer : transfers) { switch (transfer.operation) { case COPY: String valueToCopy = null; @@ -400,7 +403,7 @@ public class ConfigureAction extends AbstractServerAction { } // !!! string replacements happen only on the new values. - for (ConfigureTask.Replace replacement : replacements) { + for (Replace replacement : replacements) { if (newValues.containsKey(replacement.key)) { String toReplace = newValues.get(replacement.key); @@ -534,7 +537,7 @@ public class ConfigureAction extends AbstractServerAction { return result; } - private static String mask(ConfigureTask.Masked mask, String value) { + private static String mask(Masked mask, String value) { if (mask.mask) { return StringUtils.repeat("*", value.length()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/stack/ModuleFileUnmarshaller.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ModuleFileUnmarshaller.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ModuleFileUnmarshaller.java index aa8e17b..9e2f997 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ModuleFileUnmarshaller.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ModuleFileUnmarshaller.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.stack; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.ConfigurationXml; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.ServiceMetainfoXml; @@ -63,12 +64,13 @@ class ModuleFileUnmarshaller { try { // three classes define the top-level element "metainfo", so we need 3 contexts. JAXBContext ctx = JAXBContext.newInstance(StackMetainfoXml.class, RepositoryXml.class, - ConfigurationXml.class, UpgradePack.class); + ConfigurationXml.class, UpgradePack.class, ConfigUpgradePack.class); jaxbContexts.put(StackMetainfoXml.class, ctx); jaxbContexts.put(RepositoryXml.class, ctx); jaxbContexts.put(ConfigurationXml.class, ctx); jaxbContexts.put(UpgradePack.class, ctx); + jaxbContexts.put(ConfigUpgradePack.class, ctx); jaxbContexts.put(ServiceMetainfoXml.class, JAXBContext.newInstance(ServiceMetainfoXml.class)); } catch (JAXBException e) { throw new RuntimeException (e); http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java index 8f81b5a..c739211 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDefinitionDirectory.java @@ -37,6 +37,8 @@ public abstract class StackDefinitionDirectory { } }; + protected static final String CONFIG_UPGRADE_XML_FILENAME_PREFIX = "config-upgrade.xml"; + /** * underlying directory */ http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java index db947ca..515d031 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackDirectory.java @@ -23,6 +23,7 @@ import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.StackMetainfoXml; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.commons.io.FilenameUtils; import org.codehaus.jackson.map.ObjectMapper; @@ -98,6 +99,11 @@ public class StackDirectory extends StackDefinitionDirectory { private Map<String, UpgradePack> upgradePacks; /** + * Config delta from prev stack + */ + private ConfigUpgradePack configUpgradePack; + + /** * metainfo file representation */ private StackMetainfoXml metaInfoXml; @@ -254,6 +260,13 @@ public class StackDirectory extends StackDefinitionDirectory { } /** + * @return Config delta from prev stack or null if no config upgrade patches available + */ + public ConfigUpgradePack getConfigUpgradePack() { + return configUpgradePack; + } + + /** * Obtain the object representation of the stack role_command_order.json file * * @return object representation of the stack role_command_order.json file @@ -404,20 +417,35 @@ public class StackDirectory extends StackDefinitionDirectory { * @throws AmbariException if unable to parse stack upgrade file */ private void parseUpgradePacks(Collection<String> subDirs) throws AmbariException { - Map<String, UpgradePack> upgradeMap = new HashMap<String, UpgradePack>(); + Map<String, UpgradePack> upgradeMap = new HashMap<>(); + ConfigUpgradePack configUpgradePack = null; if (subDirs.contains(UPGRADE_PACK_FOLDER_NAME)) { File f = new File(getAbsolutePath() + File.separator + UPGRADE_PACK_FOLDER_NAME); if (f.isDirectory()) { upgradesDir = f.getAbsolutePath(); for (File upgradeFile : f.listFiles(XML_FILENAME_FILTER)) { - try { - String upgradePackName = FilenameUtils.removeExtension(upgradeFile.getName()); - UpgradePack pack = unmarshaller.unmarshal(UpgradePack.class, upgradeFile); - pack.setName(upgradePackName); - upgradeMap.put(upgradePackName, pack); - } catch (JAXBException e) { - throw new AmbariException("Unable to parse stack upgrade file at location: " + - upgradeFile.getAbsolutePath(), e); + if (upgradeFile.getName().toLowerCase().startsWith(CONFIG_UPGRADE_XML_FILENAME_PREFIX)) { + try { // Parse config upgrade pack + if (configUpgradePack == null) { + configUpgradePack = unmarshaller.unmarshal(ConfigUpgradePack.class, upgradeFile); + } else { // If user messed things up with lower/upper case filenames + throw new AmbariException(String.format("There are multiple files with name like %s" + + upgradeFile.getAbsolutePath())); + } + } catch (JAXBException e) { + throw new AmbariException("Unable to parse stack upgrade file at location: " + + upgradeFile.getAbsolutePath(), e); + } + } else { + try { + String upgradePackName = FilenameUtils.removeExtension(upgradeFile.getName()); + UpgradePack pack = unmarshaller.unmarshal(UpgradePack.class, upgradeFile); + pack.setName(upgradePackName); + upgradeMap.put(upgradePackName, pack); + } catch (JAXBException e) { + throw new AmbariException("Unable to parse stack upgrade file at location: " + + upgradeFile.getAbsolutePath(), e); + } } } } @@ -430,6 +458,13 @@ public class StackDirectory extends StackDefinitionDirectory { if (! upgradeMap.isEmpty()) { upgradePacks = upgradeMap; } + + if (configUpgradePack != null) { + this.configUpgradePack = configUpgradePack; + } else { + LOG.info("Stack '{}' doesn't contain config upgrade pack file", getPath()); + } + } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index 4b88aff..1d7da2d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -420,6 +419,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V stackInfo.setWidgetsDescriptorFileLocation(stackDirectory.getWidgetsDescriptorFilePath()); stackInfo.setUpgradesFolder(stackDirectory.getUpgradesDir()); stackInfo.setUpgradePacks(stackDirectory.getUpgradePacks()); + stackInfo.setConfigUpgradePack(stackDirectory.getConfigUpgradePack()); stackInfo.setRoleCommandOrder(stackDirectory.getRoleCommandOrder()); populateConfigurationModules(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index 87301e5..2499c4c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.ambari.server.controller.StackVersionResponse; import org.apache.ambari.server.stack.Validable; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.UpgradePack; public class StackInfo implements Comparable<StackInfo>, Validable{ @@ -48,6 +49,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ private List<PropertyInfo> properties; private Map<String, Map<String, Map<String, String>>> configTypes; private Map<String, UpgradePack> upgradePacks; + private ConfigUpgradePack configUpgradePack; private StackRoleCommandOrder roleCommandOrder; private boolean valid = true; @@ -354,6 +356,15 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } /** + * Obtain all stack upgrade packs. + * + * @return map of upgrade pack name to upgrade pack or {@code null} if no packs + */ + public Map<String, UpgradePack> getUpgradePacks() { + return upgradePacks; + } + + /** * Set upgrade packs. * * @param upgradePacks map of upgrade packs @@ -363,14 +374,22 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ } /** - * Obtain all stack upgrade packs. - * - * @return map of upgrade pack name to upgrade pack or {@code null} of no packs + * Get config upgrade pack for stack + * @return config upgrade pack for stack or null if it is + * not defined */ - public Map<String, UpgradePack> getUpgradePacks() { - return upgradePacks; + public ConfigUpgradePack getConfigUpgradePack() { + return configUpgradePack; } + /** + * Set config upgrade pack for stack + * @param configUpgradePack config upgrade pack for stack or null if it is + * not defined + */ + public void setConfigUpgradePack(ConfigUpgradePack configUpgradePack) { + this.configUpgradePack = configUpgradePack; + } @Override public int compareTo(StackInfo o) { http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index ecefe6e..442c9ed 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -47,6 +47,7 @@ import org.apache.ambari.server.stack.HostsType; import org.apache.ambari.server.stack.MasterHostResolver; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; +import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.Grouping; import org.apache.ambari.server.state.stack.upgrade.ManualTask; @@ -199,7 +200,7 @@ public class UpgradeHelper { // Note, only a Rolling Upgrade uses processing tasks. Map<String, Map<String, ProcessingComponent>> allTasks = upgradePack.getTasks(); - List<UpgradeGroupHolder> groups = new ArrayList<UpgradeGroupHolder>(); + List<UpgradeGroupHolder> groups = new ArrayList<>(); for (Grouping group : upgradePack.getGroups(context.getDirection())) { @@ -227,7 +228,7 @@ public class UpgradeHelper { // Rolling Downgrade must reverse the order of services. if (upgradePack.getType() == UpgradeType.ROLLING) { if (context.getDirection().isDowngrade() && !services.isEmpty()) { - List<UpgradePack.OrderService> reverse = new ArrayList<UpgradePack.OrderService>(services); + List<UpgradePack.OrderService> reverse = new ArrayList<>(services); Collections.reverse(reverse); services = reverse; } @@ -235,7 +236,7 @@ public class UpgradeHelper { // !!! cluster and service checks are empty here for (UpgradePack.OrderService service : services) { - + if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.containsKey(service.serviceName)) { continue; } @@ -278,7 +279,7 @@ public class UpgradeHelper { if (null != functionName) { pc = new ProcessingComponent(); pc.name = component; - pc.tasks = new ArrayList<Task>(); + pc.tasks = new ArrayList<>(); if (functionName == Type.START) { pc.tasks.add(new StartTask()); @@ -302,7 +303,7 @@ public class UpgradeHelper { // !!! revisit if needed if (!hostsType.hosts.isEmpty() && hostsType.master != null && hostsType.secondary != null) { // The order is important, first do the standby, then the active namenode. - LinkedHashSet<String> order = new LinkedHashSet<String>(); + LinkedHashSet<String> order = new LinkedHashSet<>(); order.add(hostsType.secondary); order.add(hostsType.master); @@ -398,7 +399,7 @@ public class UpgradeHelper { String result = source; - List<String> tokens = new ArrayList<String>(5); + List<String> tokens = new ArrayList<>(5); Matcher matcher = PLACEHOLDER_REGEX.matcher(source); while (matcher.find()) { tokens.add(matcher.group(1)); @@ -494,7 +495,7 @@ public class UpgradeHelper { /** * List of stages for the group */ - public List<StageWrapper> items = new ArrayList<StageWrapper>(); + public List<StageWrapper> items = new ArrayList<>(); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ConfigUpgradePack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ConfigUpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ConfigUpgradePack.java new file mode 100644 index 0000000..2896255 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ConfigUpgradePack.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.stack; + +import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Represents a pack of changes that should be applied to configs + * when upgrading from a previous stack. In other words, it's a config delta + * from prev stack + */ +@XmlRootElement(name="upgrade-config-changes") +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfigUpgradePack { + + /** + * Defines per-service config changes + */ + @XmlElementWrapper(name="services") + @XmlElement(name="service") + private List<AffectedService> services; + + /** + * Contains a cached mapping of <change id, change definition>. + */ + private Map<String, ConfigUpgradeChangeDefinition> changesById; + + private static Logger LOG = LoggerFactory.getLogger(ConfigUpgradePack.class); + + /** + * no-arg default constructor for JAXB + */ + public ConfigUpgradePack() { + } + + public ConfigUpgradePack(List<AffectedService> services) { + this.services = services; + } + + /** + * @return a list of per-service config changes. List should not be modified + * in runtime, since it will make cache stale. + */ + public List<AffectedService> getServices() { + return services; + } + + /** + * @return a map of <service name, AffectedService>. + */ + public Map<String, AffectedService> getServiceMap() { + Map<String, AffectedService> result = new HashMap<>(); + for (AffectedService service : services) { + result.put(service.name, service); + } + return result; + } + + /** + * @return a map of <change id, change definition>. Map is built once and + * cached + */ + public Map<String, ConfigUpgradeChangeDefinition> enumerateConfigChangesByID() { + if (changesById == null) { + changesById = new HashMap<>(); + for(AffectedService service : services) { + for(AffectedComponent component: service.components) { + for (ConfigUpgradeChangeDefinition changeDefinition : component.changes) { + if (changeDefinition.id == null) { + LOG.warn(String.format("Config upgrade change definition for service %s," + + " component %s has no id", service.name, component.name)); + } else if (changesById.containsKey(changeDefinition.id)) { + LOG.warn("Duplicate config upgrade change definition with ID " + + changeDefinition.id); + } + changesById.put(changeDefinition.id, changeDefinition); + } + } + } + } + return changesById; + } + + /** + * A service definition in the 'services' element. + */ + public static class AffectedService { + + @XmlAttribute + public String name; + + @XmlElement(name="component") + public List<AffectedComponent> components; + + /** + * @return a map of <component name, AffectedService> + */ + public Map<String, AffectedComponent> getComponentMap() { + Map<String, AffectedComponent> result = new HashMap<>(); + for (AffectedComponent component : components) { + result.put(component.name, component); + } + return result; + } + } + + /** + * A component definition in the 'services/service' path. + */ + public static class AffectedComponent { + + @XmlAttribute + public String name; + + @XmlElementWrapper(name="changes") + @XmlElement(name="definition") + public List<ConfigUpgradeChangeDefinition> changes; + + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java new file mode 100644 index 0000000..780f96d --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java @@ -0,0 +1,420 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.stack.upgrade; + +import com.google.gson.Gson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The {@link ConfigUpgradeChangeDefinition} represents a configuration change. This change can be + * defined with conditional statements that will only set values if a condition + * passes: + * <p/> + * + * <pre> + * {@code + * <definition> + * <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> + * <type>hive-site</type> + * <key>hive.server2.thrift.port</key> + * <value>10010</value> + * </condition> + * <condition type="hive-site" key="hive.server2.transport.mode" value="http"> + * <type>hive-site</type> + * <key>hive.server2.http.port</key> + * <value>10011</value> + * </condition> + * </definition> + * } + * </pre> + * + * It's also possible to simple set values directly without a precondition + * check. + * + * <pre> + * {@code + * <definition xsi:type="configure"> + * <type>hive-site</type> + * <set key="hive.server2.thrift.port" value="10010"/> + * <set key="foo" value="bar"/> + * <set key="foobar" value="baz"/> + * </definition> + * } + * </pre> + * + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ConfigUpgradeChangeDefinition { + + private static Logger LOG = LoggerFactory.getLogger(ConfigUpgradeChangeDefinition.class); + + /** + * The key that represents the configuration type to change (ie hdfs-site). + */ + public static final String PARAMETER_CONFIG_TYPE = "configure-task-config-type"; + + /** + * Setting key/value pairs can be several per task, so they're passed in as a + * json-ified list of objects. + */ + public static final String PARAMETER_KEY_VALUE_PAIRS = "configure-task-key-value-pairs"; + + /** + * Transfers can be several per task, so they're passed in as a json-ified + * list of objects. + */ + public static final String PARAMETER_TRANSFERS = "configure-task-transfers"; + + /** + * Replacements can be several per task, so they're passed in as a json-ified list of + * objects. + */ + public static final String PARAMETER_REPLACEMENTS = "configure-task-replacements"; + + public static final String actionVerb = "Configuring"; + + public static final Float DEFAULT_PRIORITY = 1.0f; + + /** + * Gson + */ + private Gson m_gson = new Gson(); + + /** + * An optional brief description of config changes. + */ + @XmlAttribute(name = "summary") + public String summary; + + @XmlAttribute(name = "id", required = true) + public String id; + + @XmlElement(name="type") + private String configType; + + @XmlElement(name = "set") + private List<ConfigurationKeyValue> keyValuePairs; + + @XmlElement(name = "condition") + private List<Condition> conditions; + + @XmlElement(name = "transfer") + private List<Transfer> transfers; + + @XmlElement(name="replace") + private List<Replace> replacements; + + /** + * @return the config type + */ + public String getConfigType() { + return configType; + } + + /** + * @return the list of <set key=foo value=bar/> items + */ + public List<ConfigurationKeyValue> getKeyValuePairs() { + return keyValuePairs; + } + + /** + * @return the list of conditions + */ + public List<Condition> getConditions() { + return conditions; + } + + /** + * @return the list of transfers, checking for appropriate null fields. + */ + public List<Transfer> getTransfers() { + if (null == transfers) { + return Collections.emptyList(); + } + + List<Transfer> list = new ArrayList<>(); + for (Transfer t : transfers) { + switch (t.operation) { + case COPY: + case MOVE: + if (null != t.fromKey && null != t.toKey) { + list.add(t); + } else { + LOG.warn(String.format("Transfer %s is invalid", t)); + } + break; + case DELETE: + if (null != t.deleteKey) { + list.add(t); + } else { + LOG.warn(String.format("Transfer %s is invalid", t)); + } + + break; + } + } + + return list; + } + + /** + * @return the replacement tokens, never {@code null} + */ + public List<Replace> getReplacements() { + if (null == replacements) { + return Collections.emptyList(); + } + + List<Replace> list = new ArrayList<>(); + for (Replace r : replacements) { + if (null == r.key || null == r.find || null == r.replaceWith) { + LOG.warn(String.format("Replacement %s is invalid", r)); + continue; + } + list.add(r); + } + + return list; + } + + /** + * Used for configuration updates that should mask their values from being + * printed in plain text. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class Masked { + @XmlAttribute(name = "mask") + public boolean mask = false; + } + + + /** + * A key/value pair to set in the type specified by {@link ConfigUpgradeChangeDefinition#configType} + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlType(name = "set") + public static class ConfigurationKeyValue extends Masked { + @XmlAttribute(name = "key") + public String key; + + @XmlAttribute(name = "value") + public String value; + } + + /** + * A conditional element that will only perform the configuration if the + * condition is met. + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlType(name = "condition") + public static class Condition { + @XmlAttribute(name = "type") + private String conditionConfigType; + + @XmlAttribute(name = "key") + private String conditionKey; + + @XmlAttribute(name = "value") + private String conditionValue; + + @XmlElement(name = "type") + private String configType; + + @XmlElement(name = "key") + private String key; + + @XmlElement(name = "value") + private String value; + + public String getConditionConfigType() { + return conditionConfigType; + } + + public String getConditionKey() { + return conditionKey; + } + + public String getConditionValue() { + return conditionValue; + } + + public String getConfigType() { + return configType; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + } + + /** + * A {@code transfer} element will copy, move, or delete the value of one type/key to another type/key. + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlType(name = "transfer") + public static class Transfer extends Masked { + /** + * The type of operation, such as COPY or DELETE. + */ + @XmlAttribute(name = "operation") + public TransferOperation operation; + + /** + * The configuration type to copy or move from. + */ + @XmlAttribute(name = "from-type") + public String fromType; + + /** + * The key to copy or move the configuration from. + */ + @XmlAttribute(name = "from-key") + public String fromKey; + + /** + * The key to copy the configuration value to. + */ + @XmlAttribute(name = "to-key") + public String toKey; + + /** + * The configuration key to delete, or "*" for all. + */ + @XmlAttribute(name = "delete-key") + public String deleteKey; + + /** + * If {@code true}, this will ensure that any changed properties are not + * removed during a {@link TransferOperation#DELETE}. + */ + @XmlAttribute(name = "preserve-edits") + public boolean preserveEdits = false; + + /** + * A default value to use when the configurations don't contain the + * {@link #fromKey}. + */ + @XmlAttribute(name = "default-value") + public String defaultValue; + + /** + * A data type to convert the configuration value to when the action is + * {@link TransferOperation#COPY}. + */ + @XmlAttribute(name = "coerce-to") + public TransferCoercionType coerceTo; + + // if the condition is true apply the transfer action + // only supported conditional action is DELETE + // if-type/if-key == if-value + /** + * The key to read for the if condition. + */ + @XmlAttribute(name = "if-key") + public String ifKey; + + /** + * The config type to read for the if condition. + */ + @XmlAttribute(name = "if-type") + public String ifType; + + /** + * The property value to compare against for the if condition. + */ + @XmlAttribute(name = "if-value") + public String ifValue; + + /** + * The keys to keep when the action is {@link TransferOperation#DELETE}. + */ + @XmlElement(name = "keep-key") + public List<String> keepKeys = new ArrayList<String>(); + + @Override + public String toString() { + return "Transfer{" + + "operation=" + operation + + ", fromType='" + fromType + '\'' + + ", fromKey='" + fromKey + '\'' + + ", toKey='" + toKey + '\'' + + ", deleteKey='" + deleteKey + '\'' + + ", preserveEdits=" + preserveEdits + + ", defaultValue='" + defaultValue + '\'' + + ", coerceTo=" + coerceTo + + ", ifKey='" + ifKey + '\'' + + ", ifType='" + ifType + '\'' + + ", ifValue='" + ifValue + '\'' + + ", keepKeys=" + keepKeys + + '}'; + } + } + + /** + * Used to replace strings in a key with other strings. More complex + * scenarios will be possible with regex (when needed) + */ + @XmlAccessorType(XmlAccessType.FIELD) + @XmlType(name = "replace") + public static class Replace extends Masked { + /** + * The key name + */ + @XmlAttribute(name="key") + public String key; + + /** + * The string to find + */ + @XmlAttribute(name="find") + public String find; + + /** + * The string to replace + */ + @XmlAttribute(name="replace-with") + public String replaceWith; + + @Override + public String toString() { + return "Replace{" + + "key='" + key + '\'' + + ", find='" + find + '\'' + + ", replaceWith='" + replaceWith + '\'' + + '}'; + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java index 8361ea6..a85c416 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java @@ -18,7 +18,6 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,11 +25,10 @@ import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAttribute; -import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.commons.lang.StringUtils; import org.apache.ambari.server.serveraction.upgrades.ConfigureAction; import org.apache.ambari.server.state.Cluster; @@ -40,41 +38,21 @@ import org.apache.ambari.server.state.DesiredConfig; import com.google.gson.Gson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Transfer; +import static org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Replace; +import static org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Condition; +import static org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue; /** - * The {@link ConfigureTask} represents a configuration change. This task can be - * defined with conditional statements that will only set values if a condition - * passes: + * The {@link ConfigureTask} represents a configuration change. This task + * contains id of change. Change definitions are located in a separate file (config + * upgrade pack). IDs of change definitions share the same namespace within all + * stacks * <p/> * * <pre> * {@code - * <task xsi:type="configure"> - * <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - * <type>hive-site</type> - * <key>hive.server2.thrift.port</key> - * <value>10010</value> - * </condition> - * <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - * <type>hive-site</type> - * <key>hive.server2.http.port</key> - * <value>10011</value> - * </condition> - * </task> - * } - * </pre> - * - * It's also possible to simple set values directly without a precondition - * check. - * - * <pre> - * {@code - * <task xsi:type="configure"> - * <type>hive-site</type> - * <set key="hive.server2.thrift.port" value="10010"/> - * <set key="foo" value="bar"/> - * <set key="foobar" value="baz"/> - * </task> + * <task xsi:type="configure" id="hdp_2_3_0_0-UpdateHiveConfig"/> * } * </pre> * @@ -118,29 +96,15 @@ public class ConfigureTask extends ServerSideActionTask { /** * Constructor. - * */ public ConfigureTask() { implClass = ConfigureAction.class.getName(); } - @XmlTransient private Task.Type type = Task.Type.CONFIGURE; - @XmlElement(name="type") - private String configType; - - @XmlElement(name = "set") - private List<ConfigurationKeyValue> keyValuePairs; - - @XmlElement(name = "condition") - private List<Condition> conditions; - - @XmlElement(name = "transfer") - private List<Transfer> transfers; - - @XmlElement(name="replace") - private List<Replace> replacements; + @XmlAttribute(name = "id") + public String id; /** * {@inheritDoc} @@ -161,222 +125,6 @@ public class ConfigureTask extends ServerSideActionTask { } /** - * @return the config type - */ - public String getConfigType() { - return configType; - } - - /** - * Used for configuration updates that should mask their values from being - * printed in plain text. - */ - @XmlAccessorType(XmlAccessType.FIELD) - public static class Masked { - @XmlAttribute(name = "mask") - public boolean mask = false; - } - - - /** - * A key/value pair to set in the type specified by {@link ConfigureTask#type} - */ - @XmlAccessorType(XmlAccessType.FIELD) - @XmlType(name = "set") - public static class ConfigurationKeyValue extends Masked { - @XmlAttribute(name = "key") - public String key; - - @XmlAttribute(name = "value") - public String value; - } - - /** - * A conditional element that will only perform the configuration if the - * condition is met. - */ - @XmlAccessorType(XmlAccessType.FIELD) - @XmlType(name = "condition") - public static class Condition { - @XmlAttribute(name = "type") - private String conditionConfigType; - - @XmlAttribute(name = "key") - private String conditionKey; - - @XmlAttribute(name = "value") - private String conditionValue; - - @XmlElement(name = "type") - private String configType; - - @XmlElement(name = "key") - private String key; - - @XmlElement(name = "value") - private String value; - } - - /** - * A {@code transfer} element will copy, move, or delete the value of one type/key to another type/key. - */ - @XmlAccessorType(XmlAccessType.FIELD) - @XmlType(name = "transfer") - public static class Transfer extends Masked { - /** - * The type of operation, such as COPY or DELETE. - */ - @XmlAttribute(name = "operation") - public TransferOperation operation; - - /** - * The configuration type to copy or move from. - */ - @XmlAttribute(name = "from-type") - public String fromType; - - /** - * The key to copy or move the configuration from. - */ - @XmlAttribute(name = "from-key") - public String fromKey; - - /** - * The key to copy the configuration value to. - */ - @XmlAttribute(name = "to-key") - public String toKey; - - /** - * The configuration key to delete, or "*" for all. - */ - @XmlAttribute(name = "delete-key") - public String deleteKey; - - /** - * If {@code true}, this will ensure that any changed properties are not - * removed during a {@link TransferOperation#DELETE}. - */ - @XmlAttribute(name = "preserve-edits") - public boolean preserveEdits = false; - - /** - * A default value to use when the configurations don't contain the - * {@link #fromKey}. - */ - @XmlAttribute(name = "default-value") - public String defaultValue; - - /** - * A data type to convert the configuration value to when the action is - * {@link TransferOperation#COPY}. - */ - @XmlAttribute(name = "coerce-to") - public TransferCoercionType coerceTo; - - // if the condition is true apply the transfer action - // only supported conditional action is DELETE - // if-type/if-key == if-value - /** - * The key to read for the if condition. - */ - @XmlAttribute(name = "if-key") - public String ifKey; - - /** - * The config type to read for the if condition. - */ - @XmlAttribute(name = "if-type") - public String ifType; - - /** - * The property value to compare against for the if condition. - */ - @XmlAttribute(name = "if-value") - public String ifValue; - - /** - * The keys to keep when the action is {@link TransferOperation#DELETE}. - */ - @XmlElement(name = "keep-key") - public List<String> keepKeys = new ArrayList<String>(); - } - - /** - * @return the list of transfers, checking for appropriate null fields. - */ - public List<Transfer> getTransfers() { - if (null == transfers) { - return Collections.<Transfer>emptyList(); - } - - List<Transfer> list = new ArrayList<Transfer>(); - for (Transfer t : transfers) { - switch (t.operation) { - case COPY: - case MOVE: - if (null != t.fromKey && null != t.toKey) { - list.add(t); - } - break; - case DELETE: - if (null != t.deleteKey) { - list.add(t); - } - - break; - } - } - - return list; - } - - /** - * Used to replace strings in a key with other strings. More complex - * scenarios will be possible with regex (when needed) - */ - @XmlAccessorType(XmlAccessType.FIELD) - @XmlType(name = "replace") - public static class Replace extends Masked { - /** - * The key name - */ - @XmlAttribute(name="key") - public String key; - - /** - * The string to find - */ - @XmlAttribute(name="find") - public String find; - - /** - * The string to replace - */ - @XmlAttribute(name="replace-with") - public String replaceWith; - } - - /** - * @return the replacement tokens, never {@code null} - */ - public List<Replace> getReplacements() { - if (null == replacements) { - return Collections.emptyList(); - } - - List<Replace> list = new ArrayList<Replace>(); - for (Replace r : replacements) { - if (null == r.key || null == r.find || null == r.replaceWith) { - continue; - } - list.add(r); - } - - return list; - } - - /** * Gets a map containing the following properties pertaining to the * configuration value to change: * <ul> @@ -397,21 +145,41 @@ public class ConfigureTask extends ServerSideActionTask { * handle a configuration task that is unable to set any configuration * values. */ - public Map<String, String> getConfigurationChanges(Cluster cluster) { - Map<String, String> configParameters = new HashMap<String, String>(); + public Map<String, String> getConfigurationChanges(Cluster cluster, + ConfigUpgradePack configUpgradePack) { + Map<String, String> configParameters = new HashMap<>(); + + if (this.id == null || this.id.isEmpty()) { + LOG.warn("Config task id is not defined, skipping config change"); + return configParameters; + } + + if (configUpgradePack == null) { + LOG.warn("Config upgrade pack is not defined, skipping config change"); + return configParameters; + } + + // extract config change definition, referenced by current ConfigureTask + ConfigUpgradeChangeDefinition definition = configUpgradePack.enumerateConfigChangesByID().get(this.id); + if (definition == null) { + LOG.warn(String.format("Can not resolve config change definition by id %s, " + + "skipping config change", this.id)); + return configParameters; + } // the first matched condition will win; conditions make configuration tasks singular in // the properties that can be set - when there is a condition the task will only contain // conditions + List<Condition> conditions = definition.getConditions(); if( null != conditions && !conditions.isEmpty() ){ for (Condition condition : conditions) { - String conditionConfigType = condition.conditionConfigType; - String conditionKey = condition.conditionKey; - String conditionValue = condition.conditionValue; + String conditionConfigType = condition.getConditionConfigType(); + String conditionKey = condition.getConditionKey(); + String conditionValue = condition.getConditionValue(); // always add the condition's target type just so that we have one to // return even if none of the conditions match - configParameters.put(PARAMETER_CONFIG_TYPE, condition.configType); + configParameters.put(PARAMETER_CONFIG_TYPE, condition.getConfigType()); // check the condition; if it passes, set the configuration properties // and break @@ -419,10 +187,10 @@ public class ConfigureTask extends ServerSideActionTask { conditionConfigType, conditionKey); if (conditionValue.equals(checkValue)) { - List<ConfigurationKeyValue> configurations = new ArrayList<ConfigurationKeyValue>(1); + List<ConfigurationKeyValue> configurations = new ArrayList<>(1); ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - keyValue.key = condition.key; - keyValue.value = condition.value; + keyValue.key = condition.getKey(); + keyValue.value = condition.getValue(); configurations.add(keyValue); configParameters.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, @@ -434,20 +202,21 @@ public class ConfigureTask extends ServerSideActionTask { } // this task is not a condition task, so process the other elements normally - if (null != configType) { - configParameters.put(PARAMETER_CONFIG_TYPE, configType); + if (null != definition.getConfigType()) { + configParameters.put(PARAMETER_CONFIG_TYPE, definition.getConfigType()); } // for every <set key=foo value=bar/> add it to this list - if (null != keyValuePairs && !keyValuePairs.isEmpty()) { + if (null != definition.getKeyValuePairs() && !definition.getKeyValuePairs().isEmpty()) { configParameters.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, - m_gson.toJson(keyValuePairs)); + m_gson.toJson(definition.getKeyValuePairs())); } // transfers + List<Transfer> transfers = definition.getTransfers(); if (null != transfers && !transfers.isEmpty()) { - List<Transfer> allowedTransfers = new ArrayList<Transfer>(); + List<Transfer> allowedTransfers = new ArrayList<>(); for (Transfer transfer : transfers) { if (transfer.operation == TransferOperation.DELETE) { if (StringUtils.isNotBlank(transfer.ifKey) && @@ -462,7 +231,7 @@ public class ConfigureTask extends ServerSideActionTask { if (!ifValue.toLowerCase().equals(StringUtils.lowerCase(checkValue))) { // skip adding LOG.info("Skipping property delete for {}/{} as the value {} for {}/{} is not equal to {}", - this.getConfigType(), transfer.deleteKey, checkValue, ifConfigType, ifKey, ifValue); + definition.getConfigType(), transfer.deleteKey, checkValue, ifConfigType, ifKey, ifValue); continue; } } @@ -473,6 +242,7 @@ public class ConfigureTask extends ServerSideActionTask { } // replacements + List<Replace> replacements = definition.getReplacements(); if( null != replacements && !replacements.isEmpty() ){ configParameters.put(ConfigureTask.PARAMETER_REPLACEMENTS, m_gson.toJson(replacements)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java index 9d89b7a..ec0fabf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java @@ -83,7 +83,7 @@ public class Grouping { * E.g., preupgrade, restart hosts(0), ..., restart hosts(n-1), postupgrade * @param hostsType the order collection of hosts, which may have a master and secondary * @param service the service name - * @param pc the ProcessingComponent derived from the upgrade pack. + * @param pc the AffectedComponent derived from the upgrade pack. */ @Override public void add(UpgradeContext ctx, HostsType hostsType, String service, http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java index c9c6b8c..6c0f3c7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java @@ -37,7 +37,7 @@ public abstract class StageWrapperBuilder { * @param hostsType the hosts, along with their type * @param service the service name * @param clientOnly whether the service is client only, no service checks - * @param pc the ProcessingComponent derived from the upgrade pack + * @param pc the AffectedComponent derived from the upgrade pack */ public abstract void add(UpgradeContext ctx, HostsType hostsType, String service, boolean clientOnly, ProcessingComponent pc); http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml new file mode 100644 index 0000000..fbd21a4 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml @@ -0,0 +1,55 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<upgrade-config-changes xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + + <services> + <service name="HIVE"> + <component name="HIVE_SERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_set_transport_modes"> + <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> + <type>hive-site</type> + <key>hive.server2.thrift.port</key> + <value>10010</value> + </condition> + <condition type="hive-site" key="hive.server2.transport.mode" value="http"> + <type>hive-site</type> + <key>hive.server2.http.port</key> + <value>10011</value> + </condition> + </definition> + + <definition xsi:type="configure" id="hdp_2_2_0_0_hive_server_restore_transport_mode_on_downgrade"> + <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> + <type>hive-site</type> + <key>hive.server2.thrift.port</key> + <value>10000</value> + </condition> + <condition type="hive-site" key="hive.server2.transport.mode" value="http"> + <type>hive-site</type> + <key>hive.server2.http.port</key> + <value>10001</value> + </condition> + </definition> + </changes> + </component> + </service> + </services> + +</upgrade-config-changes> http://git-wip-us.apache.org/repos/asf/ambari/blob/a67ddd27/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml index 74eb499..9900d6a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml @@ -464,18 +464,7 @@ <message>Please note that the HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> </task> - <task xsi:type="configure"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10010</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10011</value> - </condition> - </task> + <task xsi:type="configure" id="hdp_2_2_0_0_hive_server_set_transport_modes" /> </pre-upgrade> <pre-downgrade> @@ -484,18 +473,7 @@ <message>Please note that the HiveServer port will now change to 10000 if hive is using a binary transfer mode or 10001 if hive is using an http transport mode. You can use "netstat -anp | grep 1000[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message> </task> - <task xsi:type="configure"> - <condition type="hive-site" key="hive.server2.transport.mode" value="binary"> - <type>hive-site</type> - <key>hive.server2.thrift.port</key> - <value>10000</value> - </condition> - <condition type="hive-site" key="hive.server2.transport.mode" value="http"> - <type>hive-site</type> - <key>hive.server2.http.port</key> - <value>10001</value> - </condition> - </task> + <task xsi:type="configure" id="hdp_2_2_0_0_hive_server_restore_transport_mode_on_downgrade" /> </pre-downgrade> <upgrade>