SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ce953ca Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ce953ca Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ce953ca Branch: refs/heads/feature/SLIDER-151_REST_API Commit: 5ce953ca301a90e27b1cbe193a482a9cb83c3084 Parents: 32fd3f5 Author: Steve Loughran <ste...@apache.org> Authored: Wed Aug 6 16:58:27 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Aug 6 16:58:27 2014 +0100 ---------------------------------------------------------------------- .../apache/slider/common/tools/Comparators.java | 58 ++++++ .../providers/AbstractProviderService.java | 18 +- .../slider/providers/ProviderService.java | 12 +- .../server/appmaster/SliderAppMaster.java | 4 +- .../slider/server/appmaster/state/AppState.java | 134 +++++++++---- .../state/ContainerReleaseSelector.java | 38 ++++ .../MostRecentContainerReleaseSelector.java | 52 +++++ .../appmaster/state/SimpleReleaseSelector.java | 34 ++++ .../TestAppStateContainerFailure.groovy | 166 ---------------- .../appstate/TestAppStateDynamicRoles.groovy | 93 --------- .../TestAppStateRebuildOnAMRestart.groovy | 124 ------------ .../appstate/TestAppStateRolePlacement.groovy | 99 ---------- .../appstate/TestAppStateRoleRelease.groovy | 82 -------- .../TestContainerResourceAllocations.groovy | 108 ----------- .../model/appstate/TestFlexDynamicRoles.groovy | 189 ------------------ .../TestMockAppStateContainerFailure.groovy | 166 ++++++++++++++++ .../TestMockAppStateDynamicRoles.groovy | 94 +++++++++ .../TestMockAppStateFlexDynamicRoles.groovy | 190 +++++++++++++++++++ .../appstate/TestMockAppStateFlexing.groovy | 126 ++++++++++++ .../TestMockAppStateRMOperations.groovy | 176 +++++++++++++++++ .../TestMockAppStateRebuildOnAMRestart.groovy | 124 ++++++++++++ .../TestMockAppStateRolePlacement.groovy | 99 ++++++++++ .../appstate/TestMockAppStateRoleRelease.groovy | 82 ++++++++ .../TestMockContainerResourceAllocations.groovy | 108 +++++++++++ .../model/appstate/TestMockFlexing.groovy | 128 ------------- .../model/appstate/TestMockRMOperations.groovy | 176 ----------------- .../model/mock/BaseMockAppStateTest.groovy | 2 +- .../model/mock/MockProviderService.groovy | 56 +++--- .../web/rest/agent/TestAMAgentWebServices.java | 3 +- .../management/TestAMManagementWebServices.java | 3 +- 30 files changed, 1510 insertions(+), 1234 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java new file mode 100644 index 0000000..0ccca0f --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import java.io.Serializable; +import java.util.Comparator; + +public class Comparators { + + public static class LongComparator implements Comparator<Long>, Serializable { + @Override + public int compare(Long o1, Long o2) { + long result = o1 - o2; + // need to comparisons with a diff greater than integer size + if (result < 0 ) { + return -1; + } else if (result >0) { + return 1; + } + return 0; + } + } + + /** + * Little template class to reverse any comparitor + * @param <CompareType> the type that is being compared + */ + public static class ComparatorReverser<CompareType> implements Comparator<CompareType>, + Serializable { + + final Comparator<CompareType> instance; + + public ComparatorReverser(Comparator<CompareType> instance) { + this.instance = instance; + } + + @Override + public int compare(CompareType first, CompareType second) { + return instance.compare(second, first); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 7639625..7483a8d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -32,6 +32,8 @@ import org.apache.slider.core.main.ExitCodeProvider; import org.apache.slider.core.registry.info.RegisteredEndpoint; import org.apache.slider.core.registry.info.ServiceInstanceData; import org.apache.slider.server.appmaster.AMViewForProviders; +import org.apache.slider.server.appmaster.state.ContainerReleaseSelector; +import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; import org.apache.slider.server.services.registry.RegistryViewForProviders; @@ -318,11 +320,21 @@ public abstract class AbstractProviderService } @Override public void applyInitialRegistryDefinitions(URL unsecureWebAPI, - URL secureWebAPI, - ServiceInstanceData registryInstanceData) throws MalformedURLException, - IOException { + URL secureWebAPI, + ServiceInstanceData registryInstanceData) throws IOException { this.amWebAPI = unsecureWebAPI; this.registryInstanceData = registryInstanceData; } + + /** + * {@inheritDoc} + * + * + * @return The base implementation returns the most recent containers first. + */ + @Override + public ContainerReleaseSelector createContainerReleaseSelector() { + return new MostRecentContainerReleaseSelector(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index b9fa34c..4278b09 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -33,6 +33,7 @@ import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.main.ExitCodeProvider; import org.apache.slider.core.registry.info.ServiceInstanceData; import org.apache.slider.server.appmaster.AMViewForProviders; +import org.apache.slider.server.appmaster.state.ContainerReleaseSelector; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; import org.apache.slider.server.services.registry.RegistryViewForProviders; @@ -172,6 +173,13 @@ public interface ProviderService extends ProviderCore, Service, */ void applyInitialRegistryDefinitions(URL unsecureWebAPI, URL secureWebAPI, - ServiceInstanceData registryInstanceData) throws MalformedURLException, - IOException; + ServiceInstanceData registryInstanceData) + throws IOException; + + /** + * Create the container release selector for this provider...any policy + * can be implemented + * @return the selector to use for choosing containers. + */ + ContainerReleaseSelector createContainerReleaseSelector(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index b3650ca..dbf08e1 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -105,6 +105,7 @@ import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.RMOperationHandler; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; +import org.apache.slider.server.appmaster.state.SimpleReleaseSelector; import org.apache.slider.server.appmaster.web.AgentService; import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp; import org.apache.slider.server.appmaster.web.SliderAMWebApp; @@ -677,7 +678,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService fs.getFileSystem(), historyDir, liveContainers, - appInformation); + appInformation, + new SimpleReleaseSelector()); // add the AM to the list of nodes in the cluster http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 08d43db..c20c924 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -18,7 +18,9 @@ package org.apache.slider.server.appmaster.state; +import com.beust.jcommander.internal.Lists; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -63,7 +65,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -251,6 +255,8 @@ public class AppState { private int failureThreshold = 10; private String logServerURL = ""; + + private ContainerReleaseSelector containerReleaseSelector; public AppState(AbstractRecordFactory recordFactory) { this.recordFactory = recordFactory; @@ -440,22 +446,27 @@ public class AppState { * @param historyDir directory containing history files * @param liveContainers list of live containers supplied on an AM restart * @param applicationInfo + * @param releaseSelector */ public synchronized void buildInstance(AggregateConf instanceDefinition, - Configuration appmasterConfig, Configuration publishedProviderConf, + Configuration appmasterConfig, + Configuration publishedProviderConf, List<ProviderRole> providerRoles, FileSystem fs, Path historyDir, List<Container> liveContainers, - Map<String, String> applicationInfo) throws - BadClusterStateException, - BadConfigException, - IOException { + Map<String, String> applicationInfo, + SimpleReleaseSelector releaseSelector) + throws BadClusterStateException, BadConfigException, IOException { + Preconditions.checkArgument(instanceDefinition != null); + Preconditions.checkArgument(releaseSelector != null); + this.publishedProviderConf = publishedProviderConf; - this.applicationInfo = applicationInfo != null ? applicationInfo - : new HashMap<String, String>(); + this.applicationInfo = applicationInfo != null ? applicationInfo + : new HashMap<String, String>(); clientProperties = new HashMap<>(); + containerReleaseSelector = releaseSelector; Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf); @@ -465,8 +476,8 @@ public class AppState { String val = publishedProviderConf.get(key); clientProperties.put(key, val); } - - + + // set the cluster specification (once its dependency the client properties // is out the way @@ -479,15 +490,16 @@ public class AppState { } ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - + instanceDefinition.getResourceOperations(); + Set<String> roleNames = resources.getComponentNames(); for (String name : roleNames) { if (!roles.containsKey(name)) { // this is a new value log.info("Adding new role {}", name); MapOperations resComponent = resources.getComponent(name); - ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent); + ProviderRole dynamicRole = + createDynamicProviderRole(name, resComponent); buildRole(dynamicRole); providerRoles.add(dynamicRole); } @@ -498,24 +510,24 @@ public class AppState { //set the livespan MapOperations globalInternalOpts = - instanceDefinition.getInternalOperations().getGlobalOptions(); + instanceDefinition.getInternalOperations().getGlobalOptions(); startTimeThreshold = globalInternalOpts.getOptionInt( - OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, - OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE); - + OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, + OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE); + failureThreshold = globalInternalOpts.getOptionInt( - OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD, - OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD); + OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD, + OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD); initClusterStatus(); // add the roles roleHistory = new RoleHistory(providerRoles); roleHistory.onStart(fs, historyDir); - + //rebuild any live containers rebuildModelFromRestart(liveContainers); - + // any am config options to pick up logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, @@ -845,6 +857,27 @@ public class AppState { return nodes; } + + /** + * enum nodes by role ID, from either the active or live node list + * @param roleId role the container must be in + * @param active flag to indicate "use active list" rather than the smaller + * "live" list + * @return a list of nodes, may be empty + */ + public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId, + boolean active) { + List<RoleInstance> nodes = new ArrayList<>(); + Collection<RoleInstance> allRoleInstances; + allRoleInstances = active? activeContainers.values() : liveNodes.values(); + for (RoleInstance node : allRoleInstances) { + if (node.roleId == roleId) { + nodes.add(node); + } + } + return nodes; + } + /** * Build an instance map. @@ -911,19 +944,19 @@ public class AppState { throws SliderInternalStateException { ContainerId id = container.getId(); //look up the container - RoleInstance info = getActiveContainer(id); - if (info == null) { + RoleInstance instance = getActiveContainer(id); + if (instance == null) { throw new SliderInternalStateException( - "No active container with ID " + id.toString()); + "No active container with ID " + id); } //verify that it isn't already released if (containersBeingReleased.containsKey(id)) { throw new SliderInternalStateException( "Container %s already queued for release", id); } - info.released = true; - containersBeingReleased.put(id, info.container); - RoleStatus role = lookupRoleStatus(info.roleId); + instance.released = true; + containersBeingReleased.put(id, instance.container); + RoleStatus role = lookupRoleStatus(instance.roleId); role.incReleasing(); roleHistory.onContainerReleaseSubmitted(container); } @@ -1229,8 +1262,8 @@ public class AppState { if (shortLived) { roleStatus.incStartFailed(); } - - if (failedContainer!= null) { + + if (failedContainer != null) { roleHistory.onFailedContainer(failedContainer, shortLived); } @@ -1485,18 +1518,43 @@ public class AppState { // get the nodes to release int roleId = role.getKey(); - List<NodeInstance> nodesForRelease = - roleHistory.findNodesForRelease(roleId, excess); - - for (NodeInstance node : nodesForRelease) { - RoleInstance possible = findRoleInstanceOnHost(node, roleId); - if (possible == null) { - throw new SliderInternalStateException( - "Failed to find a container to release on node %s", node.hostname); + + // enum all active nodes that aren't being released + List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true); + + // cut all release-in-progress nodes + ListIterator<RoleInstance> li = containersToRelease.listIterator(); + while (li.hasNext()) { + RoleInstance next = li.next(); + if (next.released) { + li.remove(); } - containerReleaseSubmitted(possible.container); - operations.add(new ContainerReleaseOperation(possible.getId())); + } + + // warn if the desired state can't be reaced + if (containersToRelease.size() < excess) { + log.warn("Not enough nodes to release...short of {} nodes", + containersToRelease.size() - excess); + } + + // ask the release selector to sort the targets + containersToRelease = containerReleaseSelector.sortCandidates( + roleId, + containersToRelease, + excess); + + //crop to the excess + List<RoleInstance> finalCandidates = (excess < containersToRelease.size()) + ? containersToRelease.subList(0, excess) + : containersToRelease; + + + // then build up a release operation, logging each container as released + for (RoleInstance possible : finalCandidates) { + log.debug("Targeting for release: {}", possible); + containerReleaseSubmitted(possible.container); + operations.add(new ContainerReleaseOperation(possible.getId())); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java new file mode 100644 index 0000000..0cbc134 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +import java.util.List; + +/** + * Interface implemented by anything that must choose containers to release + * + */ +public interface ContainerReleaseSelector { + + /** + * Given a list of candidate containers, return a sorted version of the priority + * in which they should be released. + * @param candidates candidate list ... everything considered suitable + * @return + */ + List<RoleInstance> sortCandidates(int roleId, + List<RoleInstance> candidates, + int minimumToSelect); +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java new file mode 100644 index 0000000..9d936a1 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +import org.apache.slider.common.tools.Comparators; + +import java.io.Serializable; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * Sort the candidate list by the most recent container first. + */ +public class MostRecentContainerReleaseSelector implements ContainerReleaseSelector { + + @Override + public List<RoleInstance> sortCandidates(int roleId, + List<RoleInstance> candidates, + int minimumToSelect) { + Collections.sort(candidates, new newerThan()); + return candidates; + } + + private static class newerThan implements Comparator<RoleInstance>, Serializable { + private final Comparator<Long> innerComparator = + new Comparators.ComparatorReverser<>(new Comparators.LongComparator()); + public int compare(RoleInstance o1, RoleInstance o2) { + return innerComparator.compare(o1.createTime, o2.createTime); + + } + + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java new file mode 100644 index 0000000..b7f0e05 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.state; + +import java.util.List; + +/** + * Simplest release selector simply returns the list + */ +public class SimpleReleaseSelector implements ContainerReleaseSelector { + + @Override + public List<RoleInstance> sortCandidates(int roleId, + List<RoleInstance> candidates, + int minimumToSelect) { + return candidates; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy deleted file mode 100644 index 9c17763..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.ContainerId -import org.apache.slider.core.exceptions.SliderException -import org.apache.slider.core.exceptions.TriggerClusterTeardownException -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine -import org.apache.slider.server.appmaster.state.* -import org.junit.Test - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -@CompileStatic -@Slf4j -class TestAppStateContainerFailure extends BaseMockAppStateTest - implements MockRoles { - - @Override - String getTestName() { - return "TestAppStateContainerFailure" - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node - * @return - */ - @Override - MockYarnEngine createYarnEngine() { - return new MockYarnEngine(8000, 4) - } - - @Test - public void testShortLivedFail() throws Throwable { - - role0Status.desired = 1 - List<RoleInstance> instances = createAndStartNodes() - assert instances.size() == 1 - - RoleInstance instance = instances[0] - long created = instance.createTime - long started = instance.startTime - assert created > 0 - assert started >= created - List<ContainerId> ids = extractContainerIds(instances, 0) - - ContainerId cid = ids[0] - assert appState.isShortLived(instance) - AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1)) - assert result.roleInstance != null - assert result.containerFailed - RoleStatus status = role0Status - assert status.failed == 1 - assert status.startFailed == 1 - - //view the world - appState.getRoleHistory().dump(); - List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0) - assert queue.size() == 0 - - } - - @Test - public void testLongLivedFail() throws Throwable { - - role0Status.desired = 1 - List<RoleInstance> instances = createAndStartNodes() - assert instances.size() == 1 - - RoleInstance instance = instances[0] - instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000; - assert !appState.isShortLived(instance) - List<ContainerId> ids = extractContainerIds(instances, 0) - - ContainerId cid = ids[0] - AppState.NodeCompletionResult result = appState.onCompletedNode( - containerStatus(cid, 1)) - assert result.roleInstance != null - assert result.containerFailed - RoleStatus status = role0Status - assert status.failed == 1 - assert status.startFailed == 0 - - //view the world - appState.getRoleHistory().dump(); - List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0) - assert queue.size() == 1 - - } - - @Test - public void testNodeStartFailure() throws Throwable { - - role0Status.desired = 1 - List<RoleInstance> instances = createAndSubmitNodes() - assert instances.size() == 1 - - RoleInstance instance = instances[0] - - List<ContainerId> ids = extractContainerIds(instances, 0) - - ContainerId cid = ids[0] - appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops")) - RoleStatus status = role0Status - assert status.failed == 1 - assert status.startFailed == 1 - - - RoleHistory history = appState.roleHistory - history.dump(); - List<NodeInstance> queue = history.cloneAvailableList(0) - assert queue.size() == 0 - - NodeInstance ni = history.getOrCreateNodeInstance(instance.container) - NodeEntry re = ni.get(0) - assert re.failed == 1 - assert re.startFailed == 1 - } - - @Test - public void testRecurrentStartupFailure() throws Throwable { - - role0Status.desired = 1 - try { - for (int i = 0; i< 100; i++) { - List<RoleInstance> instances = createAndSubmitNodes() - assert instances.size() == 1 - - List<ContainerId> ids = extractContainerIds(instances, 0) - - ContainerId cid = ids[0] - log.info("$i instance $instances[0] $cid") - assert cid - appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops")) - AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid)) - assert result.containerFailed - } - fail("Cluster did not fail from too many startup failures") - } catch (TriggerClusterTeardownException teardown) { - log.info("Exception $teardown.exitCode : $teardown") - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy deleted file mode 100644 index 6d70885..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.conf.Configuration -import org.apache.slider.api.ResourceKeys -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRecordFactory -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine -import org.apache.slider.server.appmaster.state.AbstractRMOperation -import org.apache.slider.server.appmaster.state.AppState -import org.apache.slider.server.appmaster.state.RoleInstance -import org.junit.Test - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -@CompileStatic -@Slf4j -class TestAppStateDynamicRoles extends BaseMockAppStateTest - implements MockRoles { - - @Override - String getTestName() { - return "TestAppStateDynamicRoles" - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node - * @return - */ - @Override - MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4) - } - - @Override - void initApp() { - super.initApp() - appState = new AppState(new MockRecordFactory()) - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - - def instance = factory.newInstanceDefinition(0,0,0) - - def opts = [ - (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "4", - ] - - instance.resourceOperations.components["dynamic"]= opts - - - appState.buildInstance( - instance, - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, - null) - } - - @Test - public void testAllocateReleaseRealloc() throws Throwable { - - List<RoleInstance> instances = createAndStartNodes() - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - appState.getRoleHistory().dump(); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy deleted file mode 100644 index 8d7ff4f..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.yarn.api.records.Container -import org.apache.slider.api.StatusKeys -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRecordFactory -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.state.* -import org.junit.Test - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -@CompileStatic -@Slf4j -class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest - implements MockRoles { - - @Override - String getTestName() { - return "TestAppStateRebuildOnAMRestart" - } - - @Test - public void testRebuild() throws Throwable { - - int r0 = 1 - int r1 = 2 - int r2 = 3 - role0Status.desired = r0 - role1Status.desired = r1 - role2Status.desired = r2 - List<RoleInstance> instances = createAndStartNodes() - - int clusterSize = r0 + r1 + r2 - assert instances.size() == clusterSize - - //clone the list - List<RoleInstance> cloned = []; - List<Container> containers = [] - instances.each { RoleInstance elt -> - cloned.add(elt.clone() as RoleInstance) - containers.add(elt.container) - } - NodeMap nodemap = appState.roleHistory.cloneNodemap() - - // now destroy the app state - appState = new AppState(new MockRecordFactory()) - - //and rebuild - appState.buildInstance( - factory.newInstanceDefinition(r0, r1, r2), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - containers, - null) - - assert appState.getStartedCountainerCount() == clusterSize - - appState.getRoleHistory().dump(); - - //check that the app state direct structures match - List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0) - List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1) - List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2) - - assert r0 == r0live.size() - assert r1 == r1live.size() - assert r2 == r2live.size() - - //now examine the role history - NodeMap newNodemap = appState.roleHistory.cloneNodemap() - - for (NodeInstance nodeInstance : newNodemap.values()) { - String hostname = nodeInstance.hostname - NodeInstance orig = nodemap[hostname] - assertNotNull("Null entry in original nodemap for " + hostname, orig) - - for (int i = 0; i < ROLE_COUNT; i++) { - - assert (nodeInstance.getActiveRoleInstances(i) == - orig.getActiveRoleInstances(i)) - NodeEntry origRE = orig.getOrCreate(i) - NodeEntry newRE = nodeInstance.getOrCreate(i) - assert origRE.live == newRE.live - assert newRE.starting == 0 - } - } - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 0 - - def status = appState.getClusterStatus() - // verify the AM restart container count was set - String restarted = status.getInfo( - StatusKeys.INFO_CONTAINERS_AM_RESTART) - assert restarted != null; - //and that the count == 1 master + the region servers - assert Integer.parseInt(restarted) == containers.size() - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy deleted file mode 100644 index fba1ea0..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.Container -import org.apache.hadoop.yarn.client.api.AMRMClient -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.state.* -import org.junit.Test - -import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole - -/** - * Test that the app state lets you ask for nodes, get a specific host, - * release it and then get that one back again. - */ -@CompileStatic -@Slf4j -class TestAppStateRolePlacement extends BaseMockAppStateTest - implements MockRoles { - - @Override - String getTestName() { - return "TestAppStateRolePlacement" - } - - - @Test - public void testAllocateReleaseRealloc() throws Throwable { - role0Status.desired = 1 - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] - AMRMClient.ContainerRequest request = operation.request - Container allocated = engine.allocateContainer(request) - List<ContainerAssignment> assignments = []; - List<AbstractRMOperation> operations = [] - appState.onContainersAllocated([(Container)allocated], assignments, operations) - assert operations.size() == 0 - assert assignments.size() == 1 - ContainerAssignment assigned = assignments[0] - Container container = assigned.container - assert container.id == allocated.id - int roleId = assigned.role.priority - assert roleId == extractRole(request.priority) - assert assigned.role.name == ROLE0 - String containerHostname = RoleHistoryUtils.hostnameOf(container); - RoleInstance ri = roleInstance(assigned) - //tell the app it arrived - appState.containerStartSubmitted(container, ri); - assert appState.onNodeManagerContainerStarted(container.id) - assert role0Status.started == 1 - ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 0 - - //now it is surplus - role0Status.desired = 0 - ops = appState.reviewRequestAndReleaseNodes() - ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0] - - assert release.containerId == container.id - engine.execute(ops) - assert appState.onCompletedNode(containerStatus(container)).roleInstance - - //view the world - appState.getRoleHistory().dump(); - - //now ask for a new one - role0Status.desired = 1 - ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 1 - operation = (ContainerRequestOperation) ops[0] - AMRMClient.ContainerRequest request2 = operation.request - assert request2 != null - assert request2.nodes[0] == containerHostname - engine.execute(ops) - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy deleted file mode 100644 index f087a30..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.ContainerId -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine -import org.apache.slider.server.appmaster.state.AbstractRMOperation -import org.apache.slider.server.appmaster.state.RoleInstance -import org.junit.Test - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -@CompileStatic -@Slf4j -class TestAppStateRoleRelease extends BaseMockAppStateTest - implements MockRoles { - - @Override - String getTestName() { - return "TestAppStateRolePlacement" - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node - * @return - */ - @Override - MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4) - } - - @Test - public void testAllocateReleaseRealloc() throws Throwable { - /** - * Allocate to all nodes - */ - role0Status.desired = 6 - role1Status.desired = 5 - role2Status.desired = 4 - List<RoleInstance> instances = createAndStartNodes() - assert instances.size() == 15 - - //now it is surplus - role0Status.desired = 0 - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - - List<ContainerId> released = [] - engine.execute(ops, released) - List<ContainerId> ids = extractContainerIds(instances, 0) - released.each { ContainerId cid -> - assert appState.onCompletedNode(containerStatus(cid)).roleInstance - assert ids.contains(cid) - } - - //view the world - appState.getRoleHistory().dump(); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy deleted file mode 100644 index a0b1100..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.Resource -import org.apache.slider.api.ResourceKeys -import org.apache.slider.core.conf.ConfTree -import org.apache.slider.core.conf.ConfTreeOperations -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.state.AbstractRMOperation -import org.apache.slider.server.appmaster.state.ContainerRequestOperation -import org.junit.Test - -/** - * Test the container resource allocation logic - */ -@CompileStatic -@Slf4j -class TestContainerResourceAllocations extends BaseMockAppStateTest { - - @Override - String getTestName() { - "TestContainerResourceAllocations" - } - - @Test - public void testNormalAllocations() throws Throwable { - ConfTree clusterSpec = factory.newConfTree(1, 0, 0) - ConfTreeOperations cto = new ConfTreeOperations(clusterSpec) - - cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512) - cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2) - appState.updateResourceDefinitions(clusterSpec) - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 1 - ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] - Resource requirements = operation.request.capability - assert requirements.memory == 512 - assert requirements.virtualCores == 2 - } - - @Test - public void testMaxMemAllocations() throws Throwable { - ConfTree clusterSpec = factory.newConfTree(1, 0, 0) - ConfTreeOperations cto = new ConfTreeOperations(clusterSpec) - - cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, - ResourceKeys.YARN_RESOURCE_MAX) - cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2) - appState.updateResourceDefinitions(clusterSpec) - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 1 - ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] - Resource requirements = operation.request.capability - assert requirements.memory == RM_MAX_RAM - assert requirements.virtualCores == 2 - } - - @Test - public void testMaxCoreAllocations() throws Throwable { - ConfTree clusterSpec = factory.newConfTree(1, 0, 0) - ConfTreeOperations cto = new ConfTreeOperations(clusterSpec) - cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, - 512) - cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, - ResourceKeys.YARN_RESOURCE_MAX) - appState.updateResourceDefinitions(clusterSpec) - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 1 - ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] - Resource requirements = operation.request.capability - assert requirements.memory == 512 - assert requirements.virtualCores == RM_MAX_CORES - } - - @Test - public void testMaxDefaultAllocations() throws Throwable { - - ConfTree clusterSpec = factory.newConfTree(1, 0, 0) - appState.updateResourceDefinitions(clusterSpec) - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() - assert ops.size() == 1 - ContainerRequestOperation operation = (ContainerRequestOperation) ops[0] - Resource requirements = operation.request.capability - assert requirements.memory == ResourceKeys.DEF_YARN_MEMORY - assert requirements.virtualCores == ResourceKeys.DEF_YARN_CORES - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy deleted file mode 100644 index 911880a..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.slider.api.ResourceKeys -import org.apache.slider.core.conf.ConfTreeOperations -import org.apache.slider.core.exceptions.BadConfigException -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRecordFactory -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine -import org.apache.slider.server.appmaster.state.AppState -import org.apache.slider.server.avro.RoleHistoryWriter -import org.junit.Test - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -@CompileStatic -@Slf4j -class TestFlexDynamicRoles extends BaseMockAppStateTest - implements MockRoles { - - @Override - String getTestName() { - return "TestAppStateDynamicRoles" - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node - * @return - */ - @Override - MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4) - } - - @Override - void initApp() { - super.initApp() - appState = new AppState(new MockRecordFactory()) - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - - def instance = factory.newInstanceDefinition(0, 0, 0) - - def opts = [ - (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "6", - ] - - instance.resourceOperations.components["dynamic"] = opts - - - appState.buildInstance(instance, - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath, - null, null) - } - - - private ConfTreeOperations init() { - createAndStartNodes(); - def resources = appState.instanceDefinition.resources; - return new ConfTreeOperations(resources) - } - - @Test - public void testDynamicFlexAddRole() throws Throwable { - def cd = init() - def opts = [ - (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "7", - ] - - cd.components["role4"] = opts - appState.updateResourceDefinitions(cd.confTree); - createAndStartNodes(); - dumpClusterDescription("updated CD", appState.getClusterStatus()) - appState.lookupRoleStatus(7) - appState.lookupRoleStatus(6) - //gaps are still there - try { - assert null == appState.lookupRoleStatus(5) - } catch (RuntimeException expected) { - } - } - - @Test - public void testDynamicFlexAddRoleConflictingPriority() throws Throwable { - def cd = init() - def opts = [ - (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "6", - ] - - cd.components["role4"] = opts - try { - appState.updateResourceDefinitions(cd.confTree); - dumpClusterDescription("updated CD", appState.getClusterStatus()) - fail("Expected an exception") - } catch (BadConfigException expected) { - } - } - - @Test - public void testDynamicFlexDropRole() throws Throwable { - def cd = init() - cd.components.remove("dynamic"); - appState.updateResourceDefinitions(cd.confTree); - - def getCD = appState.getClusterStatus() - dumpClusterDescription("updated CD", getCD) - //status is retained for future - appState.lookupRoleStatus(6) - } - - - @Test - public void testHistorySaveFlexLoad() throws Throwable { - def cd = init() - def roleHistory = appState.roleHistory - Path history = roleHistory.saveHistory(0x0001) - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - def opts = [ - (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "7", - ] - - cd.components["role4"] = opts - appState.updateResourceDefinitions(cd.confTree); - createAndStartNodes(); - historyWriter.read(fs, history, appState.roleHistory) - } - - @Test - public void testHistoryFlexSaveLoad() throws Throwable { - def cd = init() - def opts = [ - (ResourceKeys.COMPONENT_INSTANCES): "1", - (ResourceKeys.COMPONENT_PRIORITY): "7", - ] - - cd.components["role4"] = opts - appState.updateResourceDefinitions(cd.confTree); - createAndStartNodes(); - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - def roleHistory = appState.roleHistory - Path history = roleHistory.saveHistory(0x0002) - //now reset the app state - def historyWorkDir2 = new File("target/history" + testName + "-0002") - def historyPath2 = new Path(historyWorkDir2.toURI()) - appState = new AppState(new MockRecordFactory()) - appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) - appState.buildInstance( - factory.newInstanceDefinition(0, 0, 0), - new Configuration(), - new Configuration(false), - factory.ROLES, - fs, - historyPath2, - null, null) - historyWriter.read(fs, history, appState.roleHistory) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy new file mode 100644 index 0000000..3a287bd --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.api.records.ContainerId +import org.apache.slider.core.exceptions.SliderException +import org.apache.slider.core.exceptions.TriggerClusterTeardownException +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockRoles +import org.apache.slider.server.appmaster.model.mock.MockYarnEngine +import org.apache.slider.server.appmaster.state.* +import org.junit.Test + +/** + * Test that if you have >1 role, the right roles are chosen for release. + */ +@CompileStatic +@Slf4j +class TestMockAppStateContainerFailure extends BaseMockAppStateTest + implements MockRoles { + + @Override + String getTestName() { + return "TestAppStateContainerFailure" + } + + /** + * Small cluster with multiple containers per node, + * to guarantee many container allocations on each node + * @return + */ + @Override + MockYarnEngine createYarnEngine() { + return new MockYarnEngine(8000, 4) + } + + @Test + public void testShortLivedFail() throws Throwable { + + role0Status.desired = 1 + List<RoleInstance> instances = createAndStartNodes() + assert instances.size() == 1 + + RoleInstance instance = instances[0] + long created = instance.createTime + long started = instance.startTime + assert created > 0 + assert started >= created + List<ContainerId> ids = extractContainerIds(instances, 0) + + ContainerId cid = ids[0] + assert appState.isShortLived(instance) + AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1)) + assert result.roleInstance != null + assert result.containerFailed + RoleStatus status = role0Status + assert status.failed == 1 + assert status.startFailed == 1 + + //view the world + appState.getRoleHistory().dump(); + List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0) + assert queue.size() == 0 + + } + + @Test + public void testLongLivedFail() throws Throwable { + + role0Status.desired = 1 + List<RoleInstance> instances = createAndStartNodes() + assert instances.size() == 1 + + RoleInstance instance = instances[0] + instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000; + assert !appState.isShortLived(instance) + List<ContainerId> ids = extractContainerIds(instances, 0) + + ContainerId cid = ids[0] + AppState.NodeCompletionResult result = appState.onCompletedNode( + containerStatus(cid, 1)) + assert result.roleInstance != null + assert result.containerFailed + RoleStatus status = role0Status + assert status.failed == 1 + assert status.startFailed == 0 + + //view the world + appState.getRoleHistory().dump(); + List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0) + assert queue.size() == 1 + + } + + @Test + public void testNodeStartFailure() throws Throwable { + + role0Status.desired = 1 + List<RoleInstance> instances = createAndSubmitNodes() + assert instances.size() == 1 + + RoleInstance instance = instances[0] + + List<ContainerId> ids = extractContainerIds(instances, 0) + + ContainerId cid = ids[0] + appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops")) + RoleStatus status = role0Status + assert status.failed == 1 + assert status.startFailed == 1 + + + RoleHistory history = appState.roleHistory + history.dump(); + List<NodeInstance> queue = history.cloneAvailableList(0) + assert queue.size() == 0 + + NodeInstance ni = history.getOrCreateNodeInstance(instance.container) + NodeEntry re = ni.get(0) + assert re.failed == 1 + assert re.startFailed == 1 + } + + @Test + public void testRecurrentStartupFailure() throws Throwable { + + role0Status.desired = 1 + try { + for (int i = 0; i< 100; i++) { + List<RoleInstance> instances = createAndSubmitNodes() + assert instances.size() == 1 + + List<ContainerId> ids = extractContainerIds(instances, 0) + + ContainerId cid = ids[0] + log.info("$i instance $instances[0] $cid") + assert cid + appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops")) + AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid)) + assert result.containerFailed + } + fail("Cluster did not fail from too many startup failures") + } catch (TriggerClusterTeardownException teardown) { + log.info("Exception $teardown.exitCode : $teardown") + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy new file mode 100644 index 0000000..a204bea --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.conf.Configuration +import org.apache.slider.api.ResourceKeys +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockRecordFactory +import org.apache.slider.server.appmaster.model.mock.MockRoles +import org.apache.slider.server.appmaster.model.mock.MockYarnEngine +import org.apache.slider.server.appmaster.state.AbstractRMOperation +import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.RoleInstance +import org.apache.slider.server.appmaster.state.SimpleReleaseSelector +import org.junit.Test + +/** + * Test that if you have >1 role, the right roles are chosen for release. + */ +@CompileStatic +@Slf4j +class TestMockAppStateDynamicRoles extends BaseMockAppStateTest + implements MockRoles { + + @Override + String getTestName() { + return "TestAppStateDynamicRoles" + } + + /** + * Small cluster with multiple containers per node, + * to guarantee many container allocations on each node + * @return + */ + @Override + MockYarnEngine createYarnEngine() { + return new MockYarnEngine(4, 4) + } + + @Override + void initApp() { + super.initApp() + appState = new AppState(new MockRecordFactory()) + appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) + + def instance = factory.newInstanceDefinition(0,0,0) + + def opts = [ + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY): "4", + ] + + instance.resourceOperations.components["dynamic"]= opts + + + appState.buildInstance( + instance, + new Configuration(), + new Configuration(false), + factory.ROLES, + fs, + historyPath, + null, + null, new SimpleReleaseSelector()) + } + + @Test + public void testAllocateReleaseRealloc() throws Throwable { + + List<RoleInstance> instances = createAndStartNodes() + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + appState.getRoleHistory().dump(); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy new file mode 100644 index 0000000..aa330f8 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.slider.api.ResourceKeys +import org.apache.slider.core.conf.ConfTreeOperations +import org.apache.slider.core.exceptions.BadConfigException +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockRecordFactory +import org.apache.slider.server.appmaster.model.mock.MockRoles +import org.apache.slider.server.appmaster.model.mock.MockYarnEngine +import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.SimpleReleaseSelector +import org.apache.slider.server.avro.RoleHistoryWriter +import org.junit.Test + +/** + * Test that if you have >1 role, the right roles are chosen for release. + */ +@CompileStatic +@Slf4j +class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest + implements MockRoles { + + @Override + String getTestName() { + return "TestAppStateDynamicRoles" + } + + /** + * Small cluster with multiple containers per node, + * to guarantee many container allocations on each node + * @return + */ + @Override + MockYarnEngine createYarnEngine() { + return new MockYarnEngine(4, 4) + } + + @Override + void initApp() { + super.initApp() + appState = new AppState(new MockRecordFactory()) + appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) + + def instance = factory.newInstanceDefinition(0, 0, 0) + + def opts = [ + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY): "6", + ] + + instance.resourceOperations.components["dynamic"] = opts + + + appState.buildInstance(instance, + new Configuration(), + new Configuration(false), + factory.ROLES, + fs, + historyPath, + null, null, new SimpleReleaseSelector()) + } + + + private ConfTreeOperations init() { + createAndStartNodes(); + def resources = appState.instanceDefinition.resources; + return new ConfTreeOperations(resources) + } + + @Test + public void testDynamicFlexAddRole() throws Throwable { + def cd = init() + def opts = [ + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY): "7", + ] + + cd.components["role4"] = opts + appState.updateResourceDefinitions(cd.confTree); + createAndStartNodes(); + dumpClusterDescription("updated CD", appState.getClusterStatus()) + appState.lookupRoleStatus(7) + appState.lookupRoleStatus(6) + //gaps are still there + try { + assert null == appState.lookupRoleStatus(5) + } catch (RuntimeException expected) { + } + } + + @Test + public void testDynamicFlexAddRoleConflictingPriority() throws Throwable { + def cd = init() + def opts = [ + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY): "6", + ] + + cd.components["role4"] = opts + try { + appState.updateResourceDefinitions(cd.confTree); + dumpClusterDescription("updated CD", appState.getClusterStatus()) + fail("Expected an exception") + } catch (BadConfigException expected) { + } + } + + @Test + public void testDynamicFlexDropRole() throws Throwable { + def cd = init() + cd.components.remove("dynamic"); + appState.updateResourceDefinitions(cd.confTree); + + def getCD = appState.getClusterStatus() + dumpClusterDescription("updated CD", getCD) + //status is retained for future + appState.lookupRoleStatus(6) + } + + + @Test + public void testHistorySaveFlexLoad() throws Throwable { + def cd = init() + def roleHistory = appState.roleHistory + Path history = roleHistory.saveHistory(0x0001) + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + def opts = [ + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY): "7", + ] + + cd.components["role4"] = opts + appState.updateResourceDefinitions(cd.confTree); + createAndStartNodes(); + historyWriter.read(fs, history, appState.roleHistory) + } + + @Test + public void testHistoryFlexSaveLoad() throws Throwable { + def cd = init() + def opts = [ + (ResourceKeys.COMPONENT_INSTANCES): "1", + (ResourceKeys.COMPONENT_PRIORITY): "7", + ] + + cd.components["role4"] = opts + appState.updateResourceDefinitions(cd.confTree); + createAndStartNodes(); + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + def roleHistory = appState.roleHistory + Path history = roleHistory.saveHistory(0x0002) + //now reset the app state + def historyWorkDir2 = new File("target/history" + testName + "-0002") + def historyPath2 = new Path(historyWorkDir2.toURI()) + appState = new AppState(new MockRecordFactory()) + appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES) + appState.buildInstance( + factory.newInstanceDefinition(0, 0, 0), + new Configuration(), + new Configuration(false), + factory.ROLES, + fs, + historyPath2, + null, null, new SimpleReleaseSelector()) + historyWriter.read(fs, history, appState.roleHistory) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy new file mode 100644 index 0000000..6be158e --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate + +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.api.records.Container +import org.apache.hadoop.yarn.api.records.ContainerId +import org.apache.hadoop.yarn.client.api.AMRMClient +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler +import org.apache.slider.server.appmaster.model.mock.MockRoles +import org.apache.slider.server.appmaster.state.AbstractRMOperation +import org.apache.slider.server.appmaster.state.AppState +import org.apache.slider.server.appmaster.state.ContainerAssignment +import org.apache.slider.server.appmaster.state.ContainerReleaseOperation +import org.apache.slider.server.appmaster.state.ContainerRequestOperation +import org.apache.slider.server.appmaster.state.RMOperationHandler +import org.apache.slider.server.appmaster.state.RoleInstance +import org.junit.Test + +import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority +import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole + +@Slf4j +class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles { + + @Override + String getTestName() { + return "TestMockFlexing" + } + + @Test + public void testFlexDuringLaunchPhase() throws Throwable { + role0Status.desired = 1 + + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + List<Container> allocations = engine.execute(ops) + List<ContainerAssignment> assignments = []; + List<AbstractRMOperation> releases = [] + appState.onContainersAllocated(allocations, assignments, releases) + assert assignments.size() == 1 + ContainerAssignment assigned = assignments[0] + Container target = assigned.container + RoleInstance ri = roleInstance(assigned) + + ops = appState.reviewRequestAndReleaseNodes() + assert ops.empty + + //now this is the start point. + appState.containerStartSubmitted(target, ri); + + ops = appState.reviewRequestAndReleaseNodes() + assert ops.empty + + RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id) + } + + @Test + public void testFlexBeforeAllocationPhase() throws Throwable { + role0Status.desired = 1 + + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes() + assert !ops.empty + List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes() + assert ops2.empty + } + + + @Test + public void testFlexDownTwice() throws Throwable { + int r0 = 6 + int r1 = 0 + int r2 = 0 + role0Status.desired = r0 + role1Status.desired = r1 + role2Status.desired = r2 + List<RoleInstance> instances = createAndStartNodes() + + int clusterSize = r0 + r1 + r2 + assert instances.size() == clusterSize + log.info("shrinking cluster") + r0 = 4 + role0Status.desired = r0 + List<AppState.NodeCompletionResult> completionResults = [] + instances = createStartAndStopNodes(completionResults) + assert instances.size() == 0 + // assert two nodes were released + assert completionResults.size() == 2 + + // no-op review + completionResults = [] + instances = createStartAndStopNodes(completionResults) + assert instances.size() == 0 + // assert two nodes were released + assert completionResults.size() == 0 + + + // now shrink again + role0Status.desired = r0 = 1 + completionResults = [] + instances = createStartAndStopNodes(completionResults) + assert instances.size() == 0 + // assert two nodes were released + assert completionResults.size() == 3 + + } + + +}
