Repository: ambari Updated Branches: refs/heads/branch-2.5 be1b8b5de -> 0b3058d4d
AMBARI-19448 - Role Command Order For HOU Is Different For Some Components (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0b3058d4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0b3058d4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0b3058d4 Branch: refs/heads/branch-2.5 Commit: 0b3058d4da5e295dc7b2caa6bd3d964182c318a7 Parents: be1b8b5 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Tue Jan 10 16:22:04 2017 -0500 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Wed Jan 11 13:21:17 2017 -0500 ---------------------------------------------------------------------- .../CachedRoleCommandOrderProvider.java | 23 ++- .../server/metadata/RoleCommandOrder.java | 171 ++++++++++--------- .../state/stack/StackRoleCommandOrder.java | 21 ++- .../state/stack/upgrade/HostOrderGrouping.java | 32 +++- .../stacks/HDP/2.5/role_command_order.json | 6 + .../server/metadata/RoleCommandOrderTest.java | 64 +++++++ .../stacks/HDP/2.2.0/role_command_order.json | 5 + 7 files changed, 223 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java index e2b44ca..49abc66 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metadata/CachedRoleCommandOrderProvider.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.metadata; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.Map; import org.apache.ambari.server.AmbariException; @@ -100,10 +101,24 @@ public class CachedRoleCommandOrderProvider implements RoleCommandOrderProvider RoleCommandOrder rco = rcoMap.get(clusterCacheId); if (rco == null) { rco = injector.getInstance(RoleCommandOrder.class); - rco.setHasGLUSTERFS(hasGLUSTERFS); - rco.setIsNameNodeHAEnabled(isNameNodeHAEnabled); - rco.setIsResourceManagerHAEnabled(isResourceManagerHAEnabled); - rco.initialize(cluster); + + LinkedHashSet<String> sectionKeys = new LinkedHashSet<>(); + + if (hasGLUSTERFS) { + sectionKeys.add(RoleCommandOrder.GLUSTERFS_DEPS_KEY); + } else { + sectionKeys.add(RoleCommandOrder.NO_GLUSTERFS_DEPS_KEY); + } + + if (isNameNodeHAEnabled) { + sectionKeys.add(RoleCommandOrder.NAMENODE_HA_DEPS_KEY); + } + + if (isResourceManagerHAEnabled) { + sectionKeys.add(RoleCommandOrder.RESOURCEMANAGER_HA_DEPS_KEY); + } + + rco.initialize(cluster, sectionKeys); rcoMap.put(clusterCacheId, rco); } return rco; http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java b/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java index 58675ae..234ef26 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.metadata; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; @@ -36,39 +37,38 @@ import org.apache.ambari.server.state.StackInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Sets; import com.google.inject.Inject; /** * This class is used to establish the order between two roles. This class * should not be used to determine the dependencies. */ -public class RoleCommandOrder { +public class RoleCommandOrder implements Cloneable { @Inject AmbariMetaInfo ambariMetaInfo; - private boolean hasGLUSTERFS; - private boolean isNameNodeHAEnabled; - private boolean isResourceManagerHAEnabled; - private final static Logger LOG = LoggerFactory.getLogger(RoleCommandOrder.class); + /** + * The section names used to add overides in addition to the + * {@link #GENERAL_DEPS_KEY} section. + */ + private LinkedHashSet<String> sectionKeys; + private final static String GENERAL_DEPS_KEY = "general_deps"; - private final static String GLUSTERFS_DEPS_KEY = "optional_glusterfs"; - private final static String NO_GLUSTERFS_DEPS_KEY = "optional_no_glusterfs"; - private final static String NAMENODE_HA_DEPS_KEY = "namenode_optional_ha"; - private final static String RESOURCEMANAGER_HA_DEPS_KEY = "resourcemanager_optional_ha"; - private final static String COMMENT_STR = "_comment"; + public final static String GLUSTERFS_DEPS_KEY = "optional_glusterfs"; + public final static String NO_GLUSTERFS_DEPS_KEY = "optional_no_glusterfs"; + public final static String NAMENODE_HA_DEPS_KEY = "namenode_optional_ha"; + public final static String RESOURCEMANAGER_HA_DEPS_KEY = "resourcemanager_optional_ha"; + public final static String COMMENT_STR = "_comment"; /** * Commands that are independent, role order matters */ - private static final Set<RoleCommand> independentCommands = - new HashSet<RoleCommand>() {{ - add(RoleCommand.START); - add(RoleCommand.EXECUTE); - add(RoleCommand.SERVICE_CHECK); - }}; + private static final Set<RoleCommand> independentCommands = Sets.newHashSet(RoleCommand.START, + RoleCommand.EXECUTE, RoleCommand.SERVICE_CHECK); /** * key -> blocked role command value -> set of blocker role commands. @@ -84,19 +84,23 @@ public class RoleCommandOrder { * @param blockerCommand The command on the blocking role */ private void addDependency(Role blockedRole, - RoleCommand blockedCommand, Role blockerRole, RoleCommand blockerCommand) { + RoleCommand blockedCommand, Role blockerRole, RoleCommand blockerCommand, + boolean overrideExisting) { RoleCommandPair rcp1 = new RoleCommandPair(blockedRole, blockedCommand); RoleCommandPair rcp2 = new RoleCommandPair(blockerRole, blockerCommand); - if (this.dependencies.get(rcp1) == null) { - this.dependencies.put(rcp1, new HashSet<RoleCommandPair>()); + + if (dependencies.get(rcp1) == null || overrideExisting) { + dependencies.put(rcp1, new HashSet<RoleCommandPair>()); } - this.dependencies.get(rcp1).add(rcp2); + + dependencies.get(rcp1).add(rcp2); } void addDependencies(Map<String, Object> jsonSection) { - if(jsonSection == null) // in case we don't have a certain section or role_command_order.json at all. + if(jsonSection == null) { return; - + } + for (Object blockedObj : jsonSection.keySet()) { String blocked = (String) blockedObj; if (COMMENT_STR.equals(blocked)) { @@ -108,18 +112,26 @@ public class RoleCommandOrder { String blockedRole = blockedTuple[0]; String blockedCommand = blockedTuple[1]; + // 3rd position is -OVERRIDE + boolean overrideExisting = blockedTuple.length == 3; + String [] blockerTuple = blocker.split("-"); String blockerRole = blockerTuple[0]; String blockerCommand = blockerTuple[1]; - addDependency( - Role.valueOf(blockedRole), RoleCommand.valueOf(blockedCommand), - Role.valueOf(blockerRole), RoleCommand.valueOf(blockerCommand)); + addDependency(Role.valueOf(blockedRole), RoleCommand.valueOf(blockedCommand), + Role.valueOf(blockerRole), RoleCommand.valueOf(blockerCommand), overrideExisting); } } } - public void initialize(Cluster cluster) { + @SuppressWarnings("unchecked") + public void initialize(Cluster cluster, LinkedHashSet<String> sectionKeys) { + + // in the event that initialize is called twice, ensure that we start with a + // clean RCO instance + this.sectionKeys = sectionKeys; + dependencies.clear(); StackId stackId = cluster.getCurrentStackVersion(); StackInfo stack = null; @@ -132,26 +144,15 @@ public class RoleCommandOrder { Map<String,Object> userData = stack.getRoleCommandOrder().getContent(); Map<String,Object> generalSection = (Map<String, Object>) userData.get(GENERAL_DEPS_KEY); + addDependencies(generalSection); - if (hasGLUSTERFS) { - Map<String,Object> glusterfsSection = - (Map<String, Object>) userData.get(GLUSTERFS_DEPS_KEY); - addDependencies(glusterfsSection); - } else { - Map<String,Object> noGlusterFSSection = - (Map<String, Object>) userData.get(NO_GLUSTERFS_DEPS_KEY); - addDependencies(noGlusterFSSection); - } - if (isNameNodeHAEnabled) { - Map<String,Object> NAMENODEHASection = - (Map<String, Object>) userData.get(NAMENODE_HA_DEPS_KEY); - addDependencies(NAMENODEHASection); - } - if (isResourceManagerHAEnabled) { - Map<String,Object> ResourceManagerHASection = - (Map<String, Object>) userData.get(RESOURCEMANAGER_HA_DEPS_KEY); - addDependencies(ResourceManagerHASection); + + for (String sectionKey : sectionKeys) { + Map<String, Object> section = (Map<String, Object>) userData.get(sectionKey); + + addDependencies(section); } + extendTransitiveDependency(); addMissingRestartDependencies(); } @@ -159,7 +160,7 @@ public class RoleCommandOrder { /** * Returns the dependency order. -1 => rgn1 before rgn2, 0 => they can be * parallel 1 => rgn2 before rgn1 - * + * * @param rgn1 roleGraphNode1 * @param rgn2 roleGraphNode2 */ @@ -168,11 +169,11 @@ public class RoleCommandOrder { rgn1.getCommand()); RoleCommandPair rcp2 = new RoleCommandPair(rgn2.getRole(), rgn2.getCommand()); - if ((this.dependencies.get(rcp1) != null) - && (this.dependencies.get(rcp1).contains(rcp2))) { + if ((dependencies.get(rcp1) != null) + && (dependencies.get(rcp1).contains(rcp2))) { return 1; - } else if ((this.dependencies.get(rcp2) != null) - && (this.dependencies.get(rcp2).contains(rcp1))) { + } else if ((dependencies.get(rcp2) != null) + && (dependencies.get(rcp2).contains(rcp1))) { return -1; } else if (!rgn2.getCommand().equals(rgn1.getCommand())) { return compareCommands(rgn1, rgn2); @@ -194,7 +195,7 @@ public class RoleCommandOrder { Set<RoleCommandPair> allDeps = new HashSet<RoleCommandPair>(); for (ServiceComponent sc : service.getServiceComponents().values()) { RoleCommandPair rcp = new RoleCommandPair(Role.valueOf(sc.getName()), cmd); - Set<RoleCommandPair> deps = this.dependencies.get(rcp); + Set<RoleCommandPair> deps = dependencies.get(rcp); if (deps != null) { allDeps.addAll(deps); } @@ -218,23 +219,23 @@ public class RoleCommandOrder { * A => B and B => C implies A => B,C and B => C */ private void extendTransitiveDependency() { - for (Map.Entry<RoleCommandPair, Set<RoleCommandPair>> roleCommandPairSetEntry : this.dependencies.entrySet()) { + for (Map.Entry<RoleCommandPair, Set<RoleCommandPair>> roleCommandPairSetEntry : dependencies.entrySet()) { HashSet<RoleCommandPair> visited = new HashSet<RoleCommandPair>(); HashSet<RoleCommandPair> transitiveDependencies = new HashSet<RoleCommandPair>(); - for (RoleCommandPair directlyBlockedOn : this.dependencies.get(roleCommandPairSetEntry.getKey())) { + for (RoleCommandPair directlyBlockedOn : dependencies.get(roleCommandPairSetEntry.getKey())) { visited.add(directlyBlockedOn); identifyTransitiveDependencies(directlyBlockedOn, visited, transitiveDependencies); } if (transitiveDependencies.size() > 0) { - this.dependencies.get(roleCommandPairSetEntry.getKey()).addAll(transitiveDependencies); + dependencies.get(roleCommandPairSetEntry.getKey()).addAll(transitiveDependencies); } } } private void identifyTransitiveDependencies(RoleCommandPair rcp, HashSet<RoleCommandPair> visited, HashSet<RoleCommandPair> transitiveDependencies) { - if (this.dependencies.get(rcp) != null) { - for (RoleCommandPair blockedOn : this.dependencies.get(rcp)) { + if (dependencies.get(rcp) != null) { + for (RoleCommandPair blockedOn : dependencies.get(rcp)) { if (!visited.contains(blockedOn)) { visited.add(blockedOn); transitiveDependencies.add(blockedOn); @@ -253,11 +254,11 @@ public class RoleCommandOrder { */ private void addMissingRestartDependencies() { Map<RoleCommandPair, Set<RoleCommandPair>> missingDependencies = new HashMap<>(); - for (Map.Entry<RoleCommandPair, Set<RoleCommandPair>> roleCommandPairSetEntry : this.dependencies.entrySet()) { + for (Map.Entry<RoleCommandPair, Set<RoleCommandPair>> roleCommandPairSetEntry : dependencies.entrySet()) { RoleCommandPair roleCommandPair = roleCommandPairSetEntry.getKey(); if (roleCommandPair.getCmd().equals(RoleCommand.START)) { RoleCommandPair restartPair = new RoleCommandPair(roleCommandPair.getRole(), RoleCommand.RESTART); - if (!this.dependencies.containsKey(restartPair)) { + if (!dependencies.containsKey(restartPair)) { // Assumption that if defined the RESTART deps are complete Set<RoleCommandPair> roleCommandDeps = new HashSet<>(); for (RoleCommandPair rco : roleCommandPairSetEntry.getValue()) { @@ -274,7 +275,7 @@ public class RoleCommandOrder { } } if (!missingDependencies.isEmpty()) { - this.dependencies.putAll(missingDependencies); + dependencies.putAll(missingDependencies); } } @@ -291,7 +292,7 @@ public class RoleCommandOrder { if (independentCommands.contains(rc1) && independentCommands.contains(rc2)) { return 0; } - + if (rc1.equals(RoleCommand.INSTALL)) { return -1; } else if (rc2.equals(RoleCommand.INSTALL)) { @@ -318,7 +319,7 @@ public class RoleCommandOrder { } // Check for key set match - if (!this.dependencies.keySet().equals(rco.dependencies.keySet())){ + if (!dependencies.keySet().equals(rco.dependencies.keySet())){ LOG.debug("dependency keysets differ"); return 1; } @@ -326,8 +327,8 @@ public class RoleCommandOrder { // So far so good. Since the keysets match, let's check the // actual entries against each other - for (Map.Entry<RoleCommandPair, Set<RoleCommandPair>> roleCommandPairSetEntry : this.dependencies.entrySet()) { - v1 = this.dependencies.get(roleCommandPairSetEntry.getKey()); + for (Map.Entry<RoleCommandPair, Set<RoleCommandPair>> roleCommandPairSetEntry : dependencies.entrySet()) { + v1 = dependencies.get(roleCommandPairSetEntry.getKey()); v2 = rco.dependencies.get(roleCommandPairSetEntry.getKey()); if (!v1.equals(v2)) { LOG.debug("different entry found for key (" @@ -340,28 +341,18 @@ public class RoleCommandOrder { return 0; } - public boolean isHasGLUSTERFS() { - return hasGLUSTERFS; - } - - public void setHasGLUSTERFS(boolean hasGLUSTERFS) { - this.hasGLUSTERFS = hasGLUSTERFS; - } - - public boolean isNameNodeHAEnabled() { - return isNameNodeHAEnabled; - } - - public void setIsNameNodeHAEnabled(boolean isNameNodeHAEnabled) { - this.isNameNodeHAEnabled = isNameNodeHAEnabled; - } - - public boolean isResourceManagerHAEnabled() { - return isResourceManagerHAEnabled; - } - - public void setIsResourceManagerHAEnabled(boolean isResourceManagerHAEnabled) { - this.isResourceManagerHAEnabled = isResourceManagerHAEnabled; + /** + * Gets the collection of section names that was used to initialize this + * {@link RoleCommandOrder} instance. If this instance has not been + * initialized, this will be {@code null}. + * <p/> + * The ordering of this collection is maintained. + * + * @return the section keys used to initialize this instance or {@code null} + * if it has not been initialized. + */ + public LinkedHashSet<String> getSectionKeys() { + return sectionKeys; } /** @@ -370,4 +361,16 @@ public class RoleCommandOrder { public Map<RoleCommandPair, Set<RoleCommandPair>> getDependencies() { return dependencies; } + + /** + * {@inheritDoc} + */ + @Override + public Object clone() throws CloneNotSupportedException { + RoleCommandOrder clone = (RoleCommandOrder) super.clone(); + clone.sectionKeys = new LinkedHashSet<>(sectionKeys); + clone.dependencies = new HashMap<>(dependencies); + + return clone; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackRoleCommandOrder.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackRoleCommandOrder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackRoleCommandOrder.java index 75587c4..5893de2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackRoleCommandOrder.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackRoleCommandOrder.java @@ -39,6 +39,7 @@ public class StackRoleCommandOrder { private final static String NO_GLUSTERFS_DEPS_KEY = "optional_no_glusterfs"; private final static String NAMENODE_HA_DEPS_KEY = "namenode_optional_ha"; private final static String RESOURCEMANAGER_HA_DEPS_KEY = "resourcemanager_optional_ha"; + private final static String HOST_ORDERED_UPGRADES_DEPS_KEY = "host_ordered_upgrade"; private HashMap<String, Object> content; @@ -97,8 +98,8 @@ public class StackRoleCommandOrder { HashMap<String, Object> mergedRoleCommandOrders = new HashMap<String, Object>(); HashMap<String, Object> parentData = parent.getContent(); - List<String> keys = Arrays.asList(GENERAL_DEPS_KEY, GLUSTERFS_DEPS_KEY, - NO_GLUSTERFS_DEPS_KEY, NAMENODE_HA_DEPS_KEY, RESOURCEMANAGER_HA_DEPS_KEY); + List<String> keys = Arrays.asList(GENERAL_DEPS_KEY, GLUSTERFS_DEPS_KEY, NO_GLUSTERFS_DEPS_KEY, + NAMENODE_HA_DEPS_KEY, RESOURCEMANAGER_HA_DEPS_KEY, HOST_ORDERED_UPGRADES_DEPS_KEY); for (String key : keys) { if (parentData.containsKey(key) && content.containsKey(key)) { @@ -117,12 +118,13 @@ public class StackRoleCommandOrder { if (mergeProperties) { List<String> valueList = new ArrayList<String>(); for (Object value : propertyValues) { - if (value instanceof List) + if (value instanceof List) { valueList.addAll((List<String>) value); - else - valueList.add(value.toString()); + } else { + valueList.add(value.toString()); + } } - values = valueList; + values = valueList; } result.put((String) property, values); @@ -134,7 +136,7 @@ public class StackRoleCommandOrder { mergedRoleCommandOrders.put(key, parentData.get(key)); } } - this.content = mergedRoleCommandOrders; + content = mergedRoleCommandOrders; } public void printRoleCommandOrder(Logger LOG) { @@ -152,8 +154,9 @@ public class StackRoleCommandOrder { if (depValue instanceof Collection) { StringBuffer buffer = new StringBuffer(); for (Object o : ((Collection) depValue)) { - if (buffer.length() > 0) - buffer.append(","); + if (buffer.length() > 0) { + buffer.append(","); + } buffer.append(o); } depValue = buffer.toString(); http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/HostOrderGrouping.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/HostOrderGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/HostOrderGrouping.java index abb2aab..7734731 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/HostOrderGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/HostOrderGrouping.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -144,6 +145,10 @@ public class HostOrderGrouping extends Grouping { HostRoleCommandFactory hrcFactory = upgradeContext.getHostRoleCommandFactory(); + // get a role command order instance that we can adjust for HOU since HOU + // may use a different ordering than normal start operations + RoleCommandOrder roleCommandOrder = getRoleCommandOrderForUpgrade(cluster); + for (String hostName : hosts) { // initialize the collection for all stop tasks for every component on // the host @@ -207,7 +212,6 @@ public class HostOrderGrouping extends Grouping { // now process the HRCs created so that we can create the appropriate // stage/task wrappers for the RESTARTs RoleGraphFactory roleGraphFactory = upgradeContext.getRoleGraphFactory(); - RoleCommandOrder roleCommandOrder = cluster.getRoleCommandOrder(); RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); List<Map<String, List<HostRoleCommand>>> stages = roleGraph.getOrderedHostRoleCommands( restartCommandsForHost); @@ -334,6 +338,30 @@ public class HostOrderGrouping extends Grouping { return false; } } - } + /** + * Gets a {@link RoleCommandOrder} instance initialized with + * {@code host_ordered_upgrade} overrides. + * + * @param cluster + * the cluster to get the {@link RoleCommandOrder} instance for. + * @return the order of commands for the cluster + */ + private RoleCommandOrder getRoleCommandOrderForUpgrade(Cluster cluster) { + RoleCommandOrder roleCommandOrder = cluster.getRoleCommandOrder(); + + try { + roleCommandOrder = (RoleCommandOrder) roleCommandOrder.clone(); + } catch (CloneNotSupportedException cloneNotSupportedException) { + LOG.warn("Unable to clone role command order and apply overrides for this upgrade", + cloneNotSupportedException); + } + + LinkedHashSet<String> sectionKeys = roleCommandOrder.getSectionKeys(); + sectionKeys.add("host_ordered_upgrade"); + + roleCommandOrder.initialize(cluster, sectionKeys); + return roleCommandOrder; + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json index ed99178..58dcf8d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json @@ -22,5 +22,11 @@ "optional_no_glusterfs": { "SPARK2_JOBHISTORYSERVER-START" : ["NAMENODE-START", "DATANODE-START"], "SPARK2_THRIFTSERVER-START" : ["NAMENODE-START", "DATANODE-START", "HIVE_SERVER-START"] + }, + "_comment" : "Dependencies that are used during a Host-Ordered Stack Upgrade", + "host_ordered_upgrade" : { + "DATANODE-START-OVERRIDE" : ["NAMENODE-START"], + "NODEMANAGER-START-OVERRIDE": ["RESOURCEMANAGER-START"], + "RESOURCEMANAGER-START-OVERRIDE": ["NAMENODE-START"] } } http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/test/java/org/apache/ambari/server/metadata/RoleCommandOrderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/metadata/RoleCommandOrderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/metadata/RoleCommandOrderTest.java index a2a098f..7f23f5d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/metadata/RoleCommandOrderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/metadata/RoleCommandOrderTest.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -439,6 +440,69 @@ public class RoleCommandOrderTest { Assert.assertTrue(transitiveServices.contains(hdfsService)); } + /** + * Tests the ability to override any previous mapping by using the + * {@code -OVERRIDE} placeholder. For example: + * + * <pre> + * "host_ordered_upgrade" : { + * "DATANODE-START-OVERRIDE" : ["NAMENODE-START"], + * "NODEMANAGER-START-OVERRIDE": ["RESOURCEMANAGER-START"], + * "RESOURCEMANAGER-START-OVERRIDE": ["NAMENODE-START"] + * } + * </pre> + * + * @throws IOException + */ + @Test + public void testOverride() throws Exception { + ClusterImpl cluster = createMock(ClusterImpl.class); + expect(cluster.getService("GLUSTERFS")).andReturn(null).atLeastOnce(); + expect(cluster.getClusterId()).andReturn(1L).atLeastOnce(); + Service hdfsService = createMock(Service.class); + + expect(cluster.getService("HDFS")).andReturn(hdfsService).atLeastOnce(); + expect(cluster.getService("YARN")).andReturn(null).atLeastOnce(); + expect(hdfsService.getServiceComponent("JOURNALNODE")).andReturn(null); + + // There is no rco file in this stack, should use default + expect(cluster.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.2.0")).atLeastOnce(); + + replay(cluster); + replay(hdfsService); + + RoleCommandOrder rco = roleCommandOrderProvider.getRoleCommandOrder(cluster); + + // create command pairs + RoleCommandPair startDN = new RoleCommandPair(Role.DATANODE, RoleCommand.START); + RoleCommandPair startNM = new RoleCommandPair(Role.NODEMANAGER, RoleCommand.START); + RoleCommandPair startNN = new RoleCommandPair(Role.NAMENODE, RoleCommand.START); + RoleCommandPair startRM = new RoleCommandPair(Role.RESOURCEMANAGER, RoleCommand.START); + + Set<RoleCommandPair> startDNDeps = rco.getDependencies().get(startDN); + Set<RoleCommandPair> startNMDeps = rco.getDependencies().get(startNM); + + Assert.assertNull(startDNDeps); + Assert.assertTrue(startNMDeps.contains(startDN)); + + rco = (RoleCommandOrder) rco.clone(); + LinkedHashSet<String> keys = rco.getSectionKeys(); + keys.add("host_ordered_upgrade"); + rco.initialize(cluster, keys); + + startDNDeps = rco.getDependencies().get(startDN); + startNMDeps = rco.getDependencies().get(startNM); + + // now ensure that the role orders have been modified correctly + Assert.assertTrue(startDNDeps.contains(startNN)); + Assert.assertTrue(startNMDeps.contains(startRM)); + Assert.assertFalse(startNMDeps.contains(startDN)); + Assert.assertEquals(2, startNMDeps.size()); + + verify(cluster); + verify(hdfsService); + } + private boolean dependenciesContainBlockedRole(Map<RoleCommandPair, Set<RoleCommandPair>> deps, Role blocked) { for (RoleCommandPair blockedPair : deps.keySet()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/0b3058d4/ambari-server/src/test/resources/stacks/HDP/2.2.0/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/role_command_order.json b/ambari-server/src/test/resources/stacks/HDP/2.2.0/role_command_order.json index faaee6e..0ee4e01 100644 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/role_command_order.json +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/role_command_order.json @@ -76,6 +76,11 @@ "DATANODE-UPGRADE": ["SECONDARY_NAMENODE-UPGRADE"], "HDFS_CLIENT-UPGRADE": ["DATANODE-UPGRADE"], "JOBTRACKER-UPGRADE": ["HDFS_CLIENT-UPGRADE"] + }, + "host_ordered_upgrade" : { + "DATANODE-START-OVERRIDE" : ["NAMENODE-START"], + "NODEMANAGER-START-OVERRIDE": ["RESOURCEMANAGER-START"], + "RESOURCEMANAGER-START-OVERRIDE": ["NAMENODE-START"] } }