http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java deleted file mode 100644 index 395ff22..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.server.appmaster.model.mock.MockAppState; -import org.apache.slider.server.appmaster.model.mock.MockFactory; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY; -import static org.apache.slider.server.appmaster.model.mock.MockFactory.AAROLE_2; - -/** - * Test Anti-affine placement. - */ -public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockAppStateAAPlacement.class); - - private static final int NODES = 3; - - /** - * The YARN engine has a cluster with very few nodes (3) and lots of - * containers, so if AA placement isn't working, there will be affine - * placements surfacing. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(NODES, 8); - } - - /** - * This is the simplest AA allocation: no labels, so allocate anywhere. - * @throws Throwable - */ - //@Test - public void testAllocateAANoLabel() throws Throwable { - RoleStatus aaRole = getAaRole(); - - assertTrue(cloneNodemap().size() > 0); - - // want multiple instances, so there will be iterations - aaRole.setDesired(2); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - AMRMClient.ContainerRequest request = getSingleRequest(ops); - assertFalse(request.getRelaxLocality()); - assertEquals(request.getNodes().size(), engine.getCluster() - .getClusterSize()); - assertNull(request.getRacks()); - assertNotNull(request.getCapability()); - - Container allocated = engine.allocateContainer(request); - - // notify the container ane expect - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> operations = new ArrayList<>(); - appState.onContainersAllocated(Arrays.asList(allocated), assignments, - operations); - - String host = allocated.getNodeId().getHost(); - NodeInstance hostInstance = cloneNodemap().get(host); - assertEquals(1, hostInstance.get(aaRole.getKey()).getStarting()); - assertFalse(hostInstance.canHost(aaRole.getKey(), "")); - assertFalse(hostInstance.canHost(aaRole.getKey(), null)); - - // assignment - assertEquals(1, assignments.size()); - - // verify the release matches the allocation - assertEquals(2, operations.size()); - assertNotNull(getCancel(operations, 0).getCapability().equals(allocated - .getResource())); - - // we also expect a new allocation request to have been issued - - ContainerRequest req2 = getRequest(operations, 1); - assertEquals(req2.getNodes().size(), engine.getCluster() - .getClusterSize() - 1); - - assertFalse(req2.getNodes().contains(host)); - assertFalse(request.getRelaxLocality()); - - // verify the pending couner is down - assertEquals(0L, aaRole.getAAPending()); - Container allocated2 = engine.allocateContainer(req2); - - // placement must be on a different host - assertNotEquals(allocated2.getNodeId(), allocated.getNodeId()); - - ContainerAssignment assigned = assignments.get(0); - Container container = assigned.container; - RoleInstance ri = roleInstance(assigned); - //tell the app it arrived - appState.containerStartSubmitted(container, ri); - assertNotNull(appState.onNodeManagerContainerStarted(container.getId())); - ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(0, ops.size()); - assertAllContainersAA(); - - // identify those hosts with an aa role on - Map<Integer, String> naming = appState.buildNamingMap(); - assertEquals(3, naming.size()); - - String name = aaRole.getName(); - assertEquals(name, naming.get(aaRole.getKey())); - Map<String, NodeInformation> info = - appState.getRoleHistory().getNodeInformationSnapshot(naming); - assertTrue(SliderUtils.isNotEmpty(info)); - - NodeInformation nodeInformation = info.get(host); - assertNotNull(nodeInformation); - assertTrue(SliderUtils.isNotEmpty(nodeInformation.entries)); - assertNotNull(nodeInformation.entries.get(name)); - assertEquals(1, nodeInformation.entries.get(name).live); - } - - //@Test - public void testAllocateFlexUp() throws Throwable { - RoleStatus aaRole = getAaRole(); - - // want multiple instances, so there will be iterations - aaRole.setDesired(2); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - getSingleRequest(ops); - assertEquals(1, aaRole.getRequested()); - assertEquals(1, aaRole.getAAPending()); - assertEquals(aaRole.getActualAndRequested() + aaRole - .getAAPending(), aaRole.getDesired()); - - // now trigger that flex up - aaRole.setDesired(3); - - // expect: no new reqests, pending count ++ - List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes(); - assertTrue(ops2.isEmpty()); - assertEquals(aaRole.getRunning() + aaRole.getAAPending() + - aaRole.getOutstandingAARequestCount(), aaRole.getDesired()); - - // 1 outstanding - assertEquals(0, aaRole.getRunning()); - assertTrue(aaRole.isAARequestOutstanding()); - // and one AA - assertEquals(2, aaRole.getAAPending()); - assertAllContainersAA(); - - // next iter - assertEquals(1, submitOperations(ops, EMPTY_ID_LIST, ops2).size()); - assertEquals(2, ops2.size()); - assertEquals(1, aaRole.getAAPending()); - assertAllContainersAA(); - - assertEquals(0, appState.reviewRequestAndReleaseNodes().size()); - // now trigger the next execution cycle - List<AbstractRMOperation> ops3 = new ArrayList<>(); - assertEquals(1, submitOperations(ops2, EMPTY_ID_LIST, ops3).size()); - assertEquals(2, ops3.size()); - assertEquals(0, aaRole.getAAPending()); - assertAllContainersAA(); - - } - - //@Test - public void testAllocateFlexDownDecrementsPending() throws Throwable { - RoleStatus aaRole = getAaRole(); - - // want multiple instances, so there will be iterations - aaRole.setDesired(2); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - getSingleRequest(ops); - assertEquals(1, aaRole.getAAPending()); - assertTrue(aaRole.isAARequestOutstanding()); - - // flex down so that the next request should be cancelled - aaRole.setDesired(1); - - // expect: no new requests, pending count -- - List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes(); - assertTrue(ops2.isEmpty()); - assertTrue(aaRole.isAARequestOutstanding()); - assertEquals(0, aaRole.getAAPending()); - assertAllContainersAA(); - - // next iter - submitOperations(ops, EMPTY_ID_LIST, ops2).size(); - assertEquals(1, ops2.size()); - assertAllContainersAA(); - } - - /** - * Here flex down while there is only one outstanding request. - * The outstanding flex should be cancelled - * @throws Throwable - */ - //@Test - public void testAllocateFlexDownForcesCancel() throws Throwable { - RoleStatus aaRole = getAaRole(); - - // want multiple instances, so there will be iterations - aaRole.setDesired(1); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - getSingleRequest(ops); - assertEquals(1, aaRole.getRequested()); - assertEquals(0, aaRole.getAAPending()); - assertTrue(aaRole.isAARequestOutstanding()); - - // flex down so that the next request should be cancelled - aaRole.setDesired(0); - // expect: no new requests, pending count -- - List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes(); - assertEquals(0, aaRole.getRequested()); - assertEquals(0, aaRole.getAAPending()); - assertFalse(aaRole.isAARequestOutstanding()); - assertEquals(1, ops2.size()); - getSingleCancel(ops2); - - // next iter - submitOperations(ops, EMPTY_ID_LIST, ops2).size(); - getSingleRelease(ops2); - } - - void assertAllContainersAA() { - assertAllContainersAA(getAaRole().getKey()); - } - - /** - * - * @throws Throwable - */ - //@Test - public void testAskForTooMany() throws Throwable { - RoleStatus aaRole = getAaRole(); - - describe("Ask for 1 more than the no of available nodes;" + - " expect the final request to be unsatisfied until the cluster " + - "changes size"); - //more than expected - aaRole.setDesired(NODES + 1); - List<AbstractRMOperation > operations = appState - .reviewRequestAndReleaseNodes(); - assertTrue(aaRole.isAARequestOutstanding()); - assertEquals(NODES, aaRole.getAAPending()); - for (int i = 0; i < NODES; i++) { - String iter = "Iteration " + i + " role = " + aaRole; - LOG.info(iter); - List<AbstractRMOperation > operationsOut = new ArrayList<>(); - assertEquals(1, submitOperations(operations, EMPTY_ID_LIST, - operationsOut).size()); - operations = operationsOut; - if (i + 1 < NODES) { - assertEquals(2, operations.size()); - } else { - assertEquals(1, operations.size()); - } - assertAllContainersAA(); - } - // expect an outstanding AA request to be unsatisfied - assertTrue(aaRole.getRunning() < aaRole.getDesired()); - assertEquals(0, aaRole.getRequested()); - assertFalse(aaRole.isAARequestOutstanding()); - List<Container> allocatedContainers = engine.execute(operations, - EMPTY_ID_LIST); - assertEquals(0, allocatedContainers.size()); - // in a review now, no more requests can be generated, as there is no - // space for AA placements, even though there is cluster capacity - assertEquals(0, appState.reviewRequestAndReleaseNodes().size()); - - // now do a node update (this doesn't touch the YARN engine; the node - // isn't really there) - NodeUpdatedOutcome outcome = addNewNode(); - assertEquals(cloneNodemap().size(), NODES + 1); - assertTrue(outcome.clusterChanged); - // no active calls to empty - assertTrue(outcome.operations.isEmpty()); - assertEquals(1, appState.reviewRequestAndReleaseNodes().size()); - } - - protected AppState.NodeUpdatedOutcome addNewNode() { - return updateNodes(MockFactory.INSTANCE.newNodeReport("4", NodeState - .RUNNING, "gpu")); - } - - //@Test - public void testClusterSizeChangesDuringRequestSequence() throws Throwable { - RoleStatus aaRole = getAaRole(); - describe("Change the cluster size where the cluster size changes during " + - "a test sequence."); - aaRole.setDesired(NODES + 1); - appState.reviewRequestAndReleaseNodes(); - assertTrue(aaRole.isAARequestOutstanding()); - assertEquals(NODES, aaRole.getAAPending()); - NodeUpdatedOutcome outcome = addNewNode(); - assertTrue(outcome.clusterChanged); - // one call to cancel - assertEquals(1, outcome.operations.size()); - // and on a review, one more to rebuild - assertEquals(1, appState.reviewRequestAndReleaseNodes().size()); - } - - //@Test - public void testBindingInfoMustHaveNodeMap() throws Throwable { - AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.nodeReports = null; - try { - MockAppState state = new MockAppState(bindingInfo); - fail("Expected an exception, got " + state); - } catch (IllegalArgumentException expected) { - } - } - - //@Test - public void testAMRestart() throws Throwable { - int desiredAA = 3; - getAaRole().setDesired(desiredAA); - List<RoleInstance> instances = createAndStartNodes(); - List<Container> containers = new ArrayList<>(); - for (RoleInstance instance : instances) { - containers.add(instance.container); - } - - // now destroy the app state - AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.application = factory.newApplication(0, 0, desiredAA).name( - getValidTestName()); - bindingInfo.application.getComponent(ROLE2) - .getConfiguration().setProperty(COMPONENT_PLACEMENT_POLICY, - Integer.toString(PlacementPolicy.ANTI_AFFINITY_REQUIRED)); - bindingInfo.liveContainers = containers; - appState = new MockAppState(bindingInfo); - - RoleStatus aaRole = lookupRole(AAROLE_2.name); - RoleStatus gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name); - appState.reviewRequestAndReleaseNodes(); - assertTrue(aaRole.isAntiAffinePlacement()); - assertTrue(aaRole.isAARequestOutstanding()); - - } - -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java deleted file mode 100644 index 41ff0fa..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.server.appmaster.actions.ResetFailureWindow; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockAM; -import org.apache.slider.server.appmaster.model.mock.MockAppState; -import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.ContainerOutcome; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.RoleHistory; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -public class TestMockAppStateContainerFailure extends BaseMockAppStateTest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockAppStateContainerFailure.class); - - private MockRMOperationHandler operationHandler = new - MockRMOperationHandler(); - private MockAM mockAM = new MockAM(); - - @Override - public String getTestName() { - return "TestMockAppStateContainerFailure"; - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 8000); - } - - @Override - public Application buildApplication() { - Application application = super.buildApplication(); - application.getConfiguration().setProperty( - ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "10"); - return application; - } - - //@Test - public void testShortLivedFail() throws Throwable { - - getRole0Status().setDesired(1); - List<RoleInstance> instances = createAndStartNodes(); - assertEquals(1, instances.size()); - - RoleInstance instance = instances.get(0); - long created = instance.createTime; - long started = instance.startTime; - assertTrue(created > 0); - assertTrue(started >= created); - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - - ContainerId cid = ids.get(0); - assertTrue(appState.isShortLived(instance)); - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid, 1)); - assertNotNull(result.roleInstance); - assertTrue(result.containerFailed); - RoleStatus status = getRole0Status(); - assertEquals(1, status.getFailed()); -// assertEquals(1, status.getStartFailed()); - - //view the world - appState.getRoleHistory().dump(); - List<NodeInstance> queue = appState.getRoleHistory().cloneRecentNodeList( - getRole0Status().getKey()); - assertEquals(0, queue.size()); - - } - - //@Test - public void testLongLivedFail() throws Throwable { - - getRole0Status().setDesired(1); - List<RoleInstance> instances = createAndStartNodes(); - assertEquals(1, instances.size()); - - RoleInstance instance = instances.get(0); - instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000; - assertFalse(appState.isShortLived(instance)); - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - - ContainerId cid = ids.get(0); - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid, 1)); - assertNotNull(result.roleInstance); - assertTrue(result.containerFailed); - RoleStatus status = getRole0Status(); - assertEquals(1, status.getFailed()); -// assertEquals(0, status.getStartFailed()); - - //view the world - appState.getRoleHistory().dump(); - List<NodeInstance> queue = appState.getRoleHistory().cloneRecentNodeList( - getRole0Status().getKey()); - assertEquals(1, queue.size()); - - } - - //@Test - public void testNodeStartFailure() throws Throwable { - - getRole0Status().setDesired(1); - List<RoleInstance> instances = createAndSubmitNodes(); - assertEquals(1, instances.size()); - - RoleInstance instance = instances.get(0); - - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - - ContainerId cid = ids.get(0); - appState.onNodeManagerContainerStartFailed(cid, new SliderException( - "oops")); - RoleStatus status = getRole0Status(); - assertEquals(1, status.getFailed()); -// assertEquals(1, status.getStartFailed()); - - - RoleHistory history = appState.getRoleHistory(); - history.dump(); - List<NodeInstance> queue = history.cloneRecentNodeList(getRole0Status() - .getKey()); - assertEquals(0, queue.size()); - - NodeInstance ni = history.getOrCreateNodeInstance(instance.container); - NodeEntry re = ni.get(getRole0Status().getKey()); - assertEquals(1, re.getFailed()); - assertEquals(1, re.getStartFailed()); - } - - //@Test - public void testRecurrentStartupFailure() throws Throwable { - - getRole0Status().setDesired(1); - try { - for (int i = 0; i< 100; i++) { - List<RoleInstance> instances = createAndSubmitNodes(); - assertEquals(1, instances.size()); - - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - - ContainerId cid = ids.get(0); - LOG.info("{} instance {} {}", i, instances.get(0), cid); - assertNotNull(cid); - appState.onNodeManagerContainerStartFailed(cid, - new SliderException("failure #" + i)); - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid)); - assertTrue(result.containerFailed); - } - fail("Cluster did not fail from too many startup failures"); - } catch (TriggerClusterTeardownException teardown) { - LOG.info("Exception {} : {}", teardown.getExitCode(), teardown); - } - } - - //@Test - public void testRecurrentStartupFailureWithUnlimitedFailures() throws - Throwable { - // Update instance definition to allow containers to fail any number of - // times - AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.application.getComponent(ROLE0).getConfiguration().setProperty( - ResourceKeys.CONTAINER_FAILURE_THRESHOLD, "0"); - appState = new MockAppState(bindingInfo); - - getRole0Status().setDesired(1); - try { - for (int i = 0; i < 100; i++) { - List<RoleInstance> instances = createAndSubmitNodes(); - assertEquals(1, instances.size()); - - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - - ContainerId cid = ids.get(0); - LOG.info("{} instance {} {}", i, instances.get(0), cid); - assertNotNull(cid); - appState.onNodeManagerContainerStartFailed(cid, - new SliderException("failure #" + i)); - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid)); - assertTrue(result.containerFailed); - } - } catch (TriggerClusterTeardownException teardown) { - LOG.info("Exception {} : {}", teardown.getExitCode(), teardown); - fail("Cluster failed despite " + ResourceKeys - .CONTAINER_FAILURE_THRESHOLD + " = 0"); - } - } - - //@Test - public void testRoleStatusFailureWindow() throws Throwable { - - ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); - - // initial reset - resetter.execute(mockAM, null, appState); - - getRole0Status().setDesired(1); - for (int i = 0; i < 100; i++) { - resetter.execute(mockAM, null, appState); - List<RoleInstance> instances = createAndSubmitNodes(); - assertEquals(1, instances.size()); - - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - - ContainerId cid = ids.get(0); - LOG.info("{} instance {} {}", i, instances.get(0), cid); - assertNotNull(cid); - appState.onNodeManagerContainerStartFailed( - cid, - new SliderException("failure #" + i)); - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid)); - assertTrue(result.containerFailed); - } - } - - //@Test - public void testRoleStatusFailed() throws Throwable { - RoleStatus status = getRole0Status(); - // limits exceeded - appState.incFailedContainers(status, ContainerOutcome.Failed); - assertEquals(1, status.getFailed()); - assertEquals(1L, status.getFailedRecently()); - assertEquals(0L, status.getLimitsExceeded()); - assertEquals(0L, status.getPreempted()); - assertEquals(0L, status.getDiskFailed()); - - ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); - resetter.execute(mockAM, null, appState); - assertEquals(1, status.getFailed()); - assertEquals(0L, status.getFailedRecently()); - } - - //@Test - public void testRoleStatusFailedLimitsExceeded() throws Throwable { - RoleStatus status = getRole0Status(); - // limits exceeded - appState.incFailedContainers(status, ContainerOutcome - .Failed_limits_exceeded); - assertEquals(1, status.getFailed()); - assertEquals(1L, status.getFailedRecently()); - assertEquals(1L, status.getLimitsExceeded()); - assertEquals(0L, status.getPreempted()); - assertEquals(0L, status.getDiskFailed()); - - ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); - resetter.execute(mockAM, null, appState); - assertEquals(1, status.getFailed()); - assertEquals(0L, status.getFailedRecently()); - assertEquals(1L, status.getLimitsExceeded()); - } - - - //@Test - public void testRoleStatusFailedPrempted() throws Throwable { - RoleStatus status = getRole0Status(); - // limits exceeded - appState.incFailedContainers(status, ContainerOutcome.Preempted); - assertEquals(0, status.getFailed()); - assertEquals(1L, status.getPreempted()); - assertEquals(0L, status.getFailedRecently()); - assertEquals(0L, status.getDiskFailed()); - - ResetFailureWindow resetter = new ResetFailureWindow(operationHandler); - resetter.execute(mockAM, null, appState); - assertEquals(1L, status.getPreempted()); - } - - - //@Test - public void testRoleStatusFailedNode() throws Throwable { - RoleStatus status = getRole0Status(); - // limits exceeded - appState.incFailedContainers(status, ContainerOutcome.Disk_failure); - assertEquals(1, status.getFailed()); - assertEquals(0L, status.getFailedRecently()); - assertEquals(0L, status.getLimitsExceeded()); - assertEquals(0L, status.getPreempted()); - assertEquals(1L, status.getDiskFailed()); - } - - //@Test - public void testNodeEntryCompleted() throws Throwable { - NodeEntry nodeEntry = new NodeEntry(1); - nodeEntry.containerCompleted(true, ContainerOutcome.Completed); - assertEquals(0, nodeEntry.getFailed()); - assertEquals(0, nodeEntry.getFailedRecently()); - assertEquals(0, nodeEntry.getStartFailed()); - assertEquals(0, nodeEntry.getPreempted()); - assertEquals(0, nodeEntry.getActive()); - assertTrue(nodeEntry.isAvailable()); - } - - //@Test - public void testNodeEntryFailed() throws Throwable { - NodeEntry nodeEntry = new NodeEntry(1); - nodeEntry.containerCompleted(false, ContainerOutcome.Failed); - assertEquals(1, nodeEntry.getFailed()); - assertEquals(1, nodeEntry.getFailedRecently()); - assertEquals(0, nodeEntry.getStartFailed()); - assertEquals(0, nodeEntry.getPreempted()); - assertEquals(0, nodeEntry.getActive()); - assertTrue(nodeEntry.isAvailable()); - nodeEntry.resetFailedRecently(); - assertEquals(1, nodeEntry.getFailed()); - assertEquals(0, nodeEntry.getFailedRecently()); - } - - //@Test - public void testNodeEntryLimitsExceeded() throws Throwable { - NodeEntry nodeEntry = new NodeEntry(1); - nodeEntry.containerCompleted(false, ContainerOutcome - .Failed_limits_exceeded); - assertEquals(0, nodeEntry.getFailed()); - assertEquals(0, nodeEntry.getFailedRecently()); - assertEquals(0, nodeEntry.getStartFailed()); - assertEquals(0, nodeEntry.getPreempted()); - } - - //@Test - public void testNodeEntryPreempted() throws Throwable { - NodeEntry nodeEntry = new NodeEntry(1); - nodeEntry.containerCompleted(false, ContainerOutcome.Preempted); - assertEquals(0, nodeEntry.getFailed()); - assertEquals(0, nodeEntry.getFailedRecently()); - assertEquals(0, nodeEntry.getStartFailed()); - assertEquals(1, nodeEntry.getPreempted()); - } - - //@Test - public void testNodeEntryNodeFailure() throws Throwable { - NodeEntry nodeEntry = new NodeEntry(1); - nodeEntry.containerCompleted(false, ContainerOutcome.Disk_failure); - assertEquals(1, nodeEntry.getFailed()); - assertEquals(1, nodeEntry.getFailedRecently()); - assertEquals(0, nodeEntry.getStartFailed()); - assertEquals(0, nodeEntry.getPreempted()); - } - - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java deleted file mode 100644 index cbef2be..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDependencies.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.apache.slider.server.servicemonitor.ProbeStatus; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Test for postponing container requests until dependencies are ready. - */ -public class TestMockAppStateDependencies extends BaseMockAppStateTest - implements MockRoles { - - private org.apache.slider.server.servicemonitor.Probe successProbe = - new org.apache.slider.server.servicemonitor.Probe("success", null) { - @Override - public ProbeStatus ping(ComponentInstance roleInstance) { - ProbeStatus status = new ProbeStatus(); - status.succeed(this); - return status; - } - }; - - private org.apache.slider.server.servicemonitor.Probe failureProbe = - new org.apache.slider.server.servicemonitor.Probe("failure", null) { - @Override - public ProbeStatus ping(ComponentInstance roleInstance) { - ProbeStatus status = new ProbeStatus(); - status.fail(this, new Exception()); - return status; - } - }; - - @Override - public String getTestName() { - return "TestMockAppStateDependencies"; - } - - //@Test - public void testDependencies() throws Throwable { - RoleStatus role0Status = getRole0Status(); - RoleStatus role1Status = getRole1Status(); - - // set desired instances for role0 to 1 - role0Status.setDesired(1); - // set probe for role0 to use a ping that will always succeed - role0Status.getProviderRole().probe = successProbe; - - // set desired instances for role1 to 1 - role1Status.setDesired(1); - // set role0 as a dependency of role1 - role1Status.getProviderRole().component.setDependencies(Collections - .singletonList(ROLE0)); - - // role0 has no dependencies, so its dependencies are ready - assertTrue(appState.areDependenciesReady(role0Status)); - // role1 dependency (role0) is not ready yet - assertFalse(appState.areDependenciesReady(role1Status)); - // start the single requested instance for role0 - review(ROLE0, 2); - - // role0 is still not ready because a ping has not been issued - assertFalse(appState.areDependenciesReady(role1Status)); - // issue pings - appState.monitorComponentInstances(); - // now role0 is ready - assertTrue(appState.areDependenciesReady(role1Status)); - // increase the desired containers for role0 - role0Status.setDesired(2); - // role0 is no longer ready - assertFalse(appState.areDependenciesReady(role1Status)); - // start a second instance for role0 - review(ROLE0, 2); - - // role0 is not ready because ping has not been issued for the new instance - assertFalse(appState.areDependenciesReady(role1Status)); - // issue pings - appState.monitorComponentInstances(); - // role0 is ready - assertTrue(appState.areDependenciesReady(role1Status)); - - // set probe for role0 to use a ping that will always fail - role0Status.getProviderRole().probe = failureProbe; - // issue pings - appState.monitorComponentInstances(); - // role0 is not ready (failure probe works) - assertFalse(appState.areDependenciesReady(role1Status)); - // set probe for role0 to use a ping that will always succeed - role0Status.getProviderRole().probe = successProbe; - // issue pings - appState.monitorComponentInstances(); - // role0 is ready - assertTrue(appState.areDependenciesReady(role1Status)); - - // now role1 instances can be started - review(ROLE1, 1); - } - - public void review(String expectedRole, int outstanding) throws Exception { - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - - // expect one request in the list - assertEquals(1, ops.size()); - // and in a liveness check, expected outstanding - ApplicationLivenessInformation liveness = - appState.getApplicationLivenessInformation(); - assertEquals(outstanding, liveness.requestsOutstanding); - assertFalse(liveness.allRequestsSatisfied); - - // record container allocated and verify it has the expected role - List<Container> allocations = engine.execute(ops); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> releases = new ArrayList<>(); - appState.onContainersAllocated(allocations, assignments, releases); - assertEquals(1, assignments.size()); - ContainerAssignment assigned = assignments.get(0); - Container target = assigned.container; - RoleInstance ri = roleInstance(assigned); - assertEquals(expectedRole, ri.role); - - // one fewer request outstanding - liveness = appState.getApplicationLivenessInformation(); - assertEquals(outstanding - 1, liveness.requestsOutstanding); - - // record container start submitted - appState.containerStartSubmitted(target, ri); - - // additional review results in no additional requests - ops = appState.reviewRequestAndReleaseNodes(); - assertTrue(ops.isEmpty()); - - // record container start - appState.innerOnNodeManagerContainerStarted(target.getId()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java deleted file mode 100644 index 76f3f37..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.RoleHistory; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -public class TestMockAppStateDynamicHistory extends BaseMockAppStateTest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockAppStateDynamicHistory.class); - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(8, 1); - } - - // TODO does not support adding new components dynamically - public void testDynamicRoleHistory() throws Throwable { - - String dynamic = "dynamicRole"; - long desired = 1; - int placementPolicy = PlacementPolicy.DEFAULT; - // snapshot and patch existing spec - Application application = appState.getClusterStatus(); - Component component = new Component().name(dynamic).numberOfContainers( - desired); - component.getConfiguration().setProperty(ResourceKeys - .COMPONENT_PLACEMENT_POLICY, "" + placementPolicy); - application.getComponents().add(component); - - appState.updateComponents( - Collections.singletonMap(dynamic, desired)); - - // now look at the role map - assertNotNull(appState.getRoleMap().get(dynamic)); - ProviderRole mappedRole = appState.getRoleMap().get(dynamic); - int rolePriority = mappedRole.id; - - Map<Integer, ProviderRole> priorityMap = appState.getRolePriorityMap(); - assertEquals(priorityMap.size(), 4); - ProviderRole dynamicProviderRole = priorityMap.get(rolePriority); - assertNotNull(dynamicProviderRole); - assertEquals(dynamicProviderRole.id, rolePriority); - - assertNotNull(appState.getRoleStatusMap().get(rolePriority)); - RoleStatus dynamicRoleStatus = - appState.getRoleStatusMap().get(rolePriority); - assertEquals(dynamicRoleStatus.getDesired(), desired); - - - // before allocating the nodes, fill up the capacity of some of the - // hosts - engine.getAllocator().nextIndex(); - - int targetNode = 2; - assertEquals(targetNode, engine.getAllocator().nextIndex()); - String targetHostname = engine.getCluster().nodeAt(targetNode) - .getHostname(); - - // clock is set to a small value - appState.setTime(100000); - - // allocate the nodes - List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, actions.size()); - ContainerRequestOperation action0 = (ContainerRequestOperation)actions - .get(0); - - ContainerRequest request = action0.getRequest(); - assertTrue(SliderUtils.isEmpty(request.getNodes())); - - List<ContainerId> released = new ArrayList<>(); - List<RoleInstance> allocations = submitOperations(actions, released); - processSubmissionOperations(allocations, new ArrayList<>(), released); - assertEquals(1, allocations.size()); - RoleInstance ri = allocations.get(0); - - assertEquals(ri.role, dynamic); - assertEquals(ri.roleId, rolePriority); - assertEquals(ri.host, targetHostname); - - // now look at the role history - - RoleHistory roleHistory = appState.getRoleHistory(); - List<NodeInstance> activeNodes = roleHistory.listActiveNodes( - rolePriority); - assertEquals(activeNodes.size(), 1); - NodeInstance activeNode = activeNodes.get(0); - assertNotNull(activeNode.get(rolePriority)); - NodeEntry entry8 = activeNode.get(rolePriority); - assertEquals(entry8.getActive(), 1); - - assertEquals(activeNode.hostname, targetHostname); - - NodeInstance activeNodeInstance = - roleHistory.getOrCreateNodeInstance(ri.container); - - assertEquals(activeNode, activeNodeInstance); - NodeEntry entry = activeNodeInstance.get(rolePriority); - assertNotNull(entry); - assertTrue(entry.getActive() > 0); - assertTrue(entry.getLive() > 0); - - - // now trigger a termination event on that role - - // increment time for a long-lived failure event - appState.incTime(100000); - - LOG.debug("Triggering failure"); - ContainerId cid = ri.getContainerId(); - AppState.NodeCompletionResult result = appState.onCompletedContainer( - containerStatus(cid, 1)); - assertEquals(result.roleInstance, ri); - assertTrue(result.containerFailed); - - roleHistory.dump(); - // values should have changed - assertEquals(1, entry.getFailed()); - assertEquals(0, entry.getStartFailed()); - assertEquals(0, entry.getActive()); - assertEquals(0, entry.getLive()); - - - List<NodeInstance> nodesForRoleId = - roleHistory.getRecentNodesForRoleId(rolePriority); - assertNotNull(nodesForRoleId); - - // make sure new nodes will default to a different host in the engine - assertTrue(targetNode < engine.getAllocator().nextIndex()); - - actions = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, actions.size()); - ContainerRequestOperation action1 = (ContainerRequestOperation) actions - .get(0); - ContainerRequest request1 = action1.getRequest(); - assertTrue(SliderUtils.isNotEmpty(request1.getNodes())); - } - - //@Test(expected = BadConfigException.class) - public void testRoleHistoryRoleAdditions() throws Throwable { - MockRoleHistory roleHistory = new MockRoleHistory(new ArrayList<>()); - roleHistory.addNewRole(new RoleStatus(new ProviderRole("one", 1))); - roleHistory.addNewRole(new RoleStatus(new ProviderRole("two", 1))); - roleHistory.dump(); - } - - //@Test(expected = BadConfigException.class) - public void testRoleHistoryRoleStartupConflict() throws Throwable { - MockRoleHistory roleHistory = new MockRoleHistory(Arrays.asList( - new ProviderRole("one", 1), new ProviderRole("two", 1) - )); - roleHistory.dump(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java deleted file mode 100644 index 5669d2d..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.AppState.NodeCompletionResult; -import org.apache.slider.server.appmaster.state.ContainerPriority; -import org.apache.slider.server.appmaster.state.RoleHistoryUtils; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -import static org.apache.slider.server.appmaster.model.mock.MockFactory.NODE_FAILURE_THRESHOLD; - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -public class TestMockAppStateDynamicRoles extends BaseMockAppStateTest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockAppStateDynamicRoles.class); - private static final String ROLE4 = "4"; - private static final String ROLE5 = "5"; - - @Override - public String getTestName() { - return "TestMockAppStateDynamicRoles"; - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(8, 2); - } - - @Override - public Application buildApplication() { - Application application = super.buildApplication(); - - Component component = new Component().name(ROLE4).numberOfContainers(1L); - component.getConfiguration().setProperty(ResourceKeys - .NODE_FAILURE_THRESHOLD, Integer.toString(3)); - application.getComponents().add(component); - - component = new Component().name(ROLE5).numberOfContainers(1L); - component.getConfiguration().setProperty(ResourceKeys - .COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy.STRICT)); - application.getComponents().add(component); - - return application; - } - - //@Test - public void testAllocateReleaseRealloc() throws Throwable { - - createAndStartNodes(); - appState.reviewRequestAndReleaseNodes(); - appState.getRoleHistory().dump(); - } - - /** - * Find all allocations for a specific role. - * @param role role Id/priority - * @param actions source list - * @return found list - */ - List<ContainerRequestOperation> findAllocationsForRole(int role, - List<AbstractRMOperation> actions) { - List<ContainerRequestOperation> ops = new ArrayList<>(); - for (AbstractRMOperation op : actions) { - if (op instanceof ContainerRequestOperation && role == - ContainerPriority.extractRole(((ContainerRequestOperation) op) - .getRequest().getPriority())) { - ops.add((ContainerRequestOperation) op); - } - } - return ops; - } - - //@Test - public void testStrictPlacementInitialRequest() throws Throwable { - LOG.info("Initial engine state = {}", engine); - List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes(); - assertEquals(2, actions.size()); - - // neither have locality at this point - assertRelaxLocalityFlag(appState.lookupRoleStatus(ROLE4).getKey(), null, - true, actions); - assertRelaxLocalityFlag(appState.lookupRoleStatus(ROLE5).getKey(), null, - true, actions); - } - - //@Test - public void testPolicyPropagation() throws Throwable { - assertEquals(0, (appState.lookupRoleStatus(ROLE4).getPlacementPolicy() & - PlacementPolicy.STRICT)); - assertNotEquals(0, (appState.lookupRoleStatus(ROLE5).getPlacementPolicy() & - PlacementPolicy.STRICT)); - - } - - //@Test - public void testNodeFailureThresholdPropagation() throws Throwable { - assertEquals(3, appState.lookupRoleStatus(ROLE4).getNodeFailureThreshold()); - assertEquals(NODE_FAILURE_THRESHOLD, appState.lookupRoleStatus(ROLE5) - .getNodeFailureThreshold()); - } - - //@Test - public void testLaxPlacementSecondRequestRole4() throws Throwable { - LOG.info("Initial engine state = {}", engine); - RoleStatus role4 = appState.lookupRoleStatus(ROLE4); - RoleStatus role5 = appState.lookupRoleStatus(ROLE5); - role4.setDesired(1); - role5.setDesired(0); - - List<RoleInstance> instances = createStartAndStopNodes(new ArrayList<>()); - assertEquals(1, instances.size()); - - int id = appState.lookupRoleStatus(ROLE4).getKey(); - RoleInstance instanceA = null; - for (RoleInstance instance : instances) { - if (instance.roleId == id) { - instanceA = instance; - } - } - assertNotNull(instanceA); - String hostname = RoleHistoryUtils.hostnameOf(instanceA.container); - - LOG.info("Allocated engine state = {}", engine); - assertEquals(1, engine.containerCount()); - - assertEquals(1, role4.getRunning()); - // shrinking cluster - - role4.setDesired(0); - appState.lookupRoleStatus(ROLE4).setDesired(0); - List<NodeCompletionResult> completionResults = new ArrayList<>(); - createStartAndStopNodes(completionResults); - assertEquals(0, engine.containerCount()); - assertEquals(1, completionResults.size()); - - // expanding: expect hostnames now - role4.setDesired(1); - List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, actions.size()); - - ContainerRequestOperation cro = (ContainerRequestOperation) actions.get(0); - List<String> nodes = cro.getRequest().getNodes(); - assertEquals(1, nodes.size()); - assertEquals(hostname, nodes.get(0)); - } - - //@Test - public void testStrictPlacementSecondRequestRole5() throws Throwable { - LOG.info("Initial engine state = {}", engine); - RoleStatus role4 = appState.lookupRoleStatus(ROLE4); - RoleStatus role5 = appState.lookupRoleStatus(ROLE5); - role4.setDesired(0); - role5.setDesired(1); - - List<RoleInstance> instances = createStartAndStopNodes(new ArrayList<>()); - assertEquals(1, instances.size()); - - int id = appState.lookupRoleStatus(ROLE5).getKey(); - RoleInstance instanceA = null; - for (RoleInstance instance : instances) { - if (instance.roleId == id) { - instanceA = instance; - } - } - assertNotNull(instanceA); - String hostname = RoleHistoryUtils.hostnameOf(instanceA.container); - - LOG.info("Allocated engine state = {}", engine); - assertEquals(1, engine.containerCount()); - - assertEquals(1, role5.getRunning()); - - // shrinking cluster - role5.setDesired(0); - List<NodeCompletionResult> completionResults = new ArrayList<>(); - createStartAndStopNodes(completionResults); - assertEquals(0, engine.containerCount()); - assertEquals(1, completionResults.size()); - assertEquals(0, role5.getRunning()); - - role5.setDesired(1); - List<AbstractRMOperation> actions = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, actions.size()); - assertRelaxLocalityFlag(id, "", false, actions); - ContainerRequestOperation cro = (ContainerRequestOperation) actions.get(0); - List<String> nodes = cro.getRequest().getNodes(); - assertEquals(1, nodes.size()); - assertEquals(hostname, nodes.get(0)); - } - - public void assertRelaxLocalityFlag( - int role, - String expectedHost, - boolean expectedRelaxFlag, - List<AbstractRMOperation> actions) { - List<ContainerRequestOperation> requests = findAllocationsForRole( - role, actions); - assertEquals(1, requests.size()); - ContainerRequestOperation req = requests.get(0); - assertEquals(expectedRelaxFlag, req.getRequest().getRelaxLocality()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java deleted file mode 100644 index 3304da1..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.fs.Path; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.core.exceptions.SliderInternalStateException; -import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockAppState; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; -import org.apache.slider.server.appmaster.state.RoleHistory; -import org.apache.slider.server.avro.LoadedRoleHistory; -import org.apache.slider.server.avro.RoleHistoryWriter; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.Collections; - -/** - * Test that if you have more than one role, the right roles are chosen for - * release. - */ -public class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockAppStateFlexDynamicRoles.class); - - @Override - public String getTestName() { - return "TestMockAppStateFlexDynamicRoles"; - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4); - } - - @Override - public AppStateBindingInfo buildBindingInfo() throws IOException { - AppStateBindingInfo bindingInfo = super.buildBindingInfo(); - bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); - return bindingInfo; - } - - @Override - public Application buildApplication() { - Application application = super.buildApplication(); - Component component = new Component().name("dynamic-6") - .numberOfContainers(1L); - application.getComponents().add(component); - - return application; - } - - @Before - public void init() - throws TriggerClusterTeardownException, SliderInternalStateException { - createAndStartNodes(); - } - - // TODO does not support adding new components dynamically - public void testDynamicFlexAddRole() throws Throwable { - Application application = appState.getClusterStatus(); - Component component = new Component().name("dynamicAdd7") - .numberOfContainers(1L); - application.getComponents().add(component); - appState.updateComponents(Collections.singletonMap(component.getName(), - component.getNumberOfContainers())); - createAndStartNodes(); - appState.lookupRoleStatus("dynamicAdd7"); - } - - //@Test - public void testDynamicFlexDropRole() throws Throwable { - appState.updateComponents(Collections.singletonMap("dynamic-6", 0L)); - //status is retained for future - appState.lookupRoleStatus("dynamic-6"); - } - - - //@Test - public void testHistorySaveFlexLoad() throws Throwable { - Application application = appState.getClusterStatus(); - RoleHistory roleHistory = appState.getRoleHistory(); - Path history = roleHistory.saveHistory(0x0001); - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - Component component = new Component().name("HistorySaveFlexLoad") - .numberOfContainers(1L); - application.getComponents().add(component); - - appState.updateComponents(Collections.singletonMap(component.getName(), - component.getNumberOfContainers())); - createAndStartNodes(); - LoadedRoleHistory loadedRoleHistory = - historyWriter.read(fs, history); - assertEquals(0, appState.getRoleHistory().rebuild(loadedRoleHistory)); - } - - //@Test - public void testHistoryFlexSaveResetLoad() throws Throwable { - Application application = appState.getClusterStatus(); - Component component = new Component().name("HistoryFlexSaveLoad") - .numberOfContainers(1L); - application.getComponents().add(component); - - appState.updateComponents(Collections.singletonMap(component.getName(), - component.getNumberOfContainers())); - createAndStartNodes(); - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - RoleHistory roleHistory = appState.getRoleHistory(); - Path history = roleHistory.saveHistory(0x0002); - //now reset the app state - File historyWorkDir2 = new File("target/history" + getTestName() + - "-0002"); - Path historyPath2 = new Path(historyWorkDir2.toURI()); - appState = new MockAppState(); - AppStateBindingInfo binding2 = buildBindingInfo(); - binding2.application = factory.newApplication(0, 0, 0) - .name(getValidTestName()); - binding2.historyPath = historyPath2; - appState.buildInstance(binding2); - // on this read there won't be the right number of roles - LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history); - assertEquals(0, appState.getRoleHistory().rebuild(loadedRoleHistory)); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e00bb2ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java deleted file mode 100644 index dbad599..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.CancelSingleRequest; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * Test app state flexing. - */ -public class TestMockAppStateFlexing extends BaseMockAppStateTest implements - MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(BaseMockAppStateTest.class); - - @Override - public String getTestName() { - return "TestMockAppStateFlexing"; - } - - //@Test - public void testFlexDuringLaunchPhase() throws Throwable { - - // ask for one instance of role0 - getRole0Status().setDesired(1); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - - // at this point there's now one request in the list - assertEquals(1, ops.size()); - // and in a liveness check, one outstanding - ApplicationLivenessInformation liveness = - appState.getApplicationLivenessInformation(); - assertEquals(1, liveness.requestsOutstanding); - assertFalse(liveness.allRequestsSatisfied); - - List<Container> allocations = engine.execute(ops); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> releases = new ArrayList<>(); - appState.onContainersAllocated(allocations, assignments, releases); - assertEquals(1, assignments.size()); - ContainerAssignment assigned = assignments.get(0); - Container target = assigned.container; - RoleInstance ri = roleInstance(assigned); - - ops = appState.reviewRequestAndReleaseNodes(); - assertTrue(ops.isEmpty()); - - liveness = appState.getApplicationLivenessInformation(); - assertEquals(0, liveness.requestsOutstanding); - assertTrue(liveness.allRequestsSatisfied); - - //now this is the start point. - appState.containerStartSubmitted(target, ri); - - ops = appState.reviewRequestAndReleaseNodes(); - assertTrue(ops.isEmpty()); - - appState.innerOnNodeManagerContainerStarted(target.getId()); - } - - //@Test - public void testFlexBeforeAllocationPhase() throws Throwable { - getRole0Status().setDesired(1); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertFalse(ops.isEmpty()); - - // second scan will find the first run outstanding, so not re-issue - // any more container requests - List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes(); - assertTrue(ops2.isEmpty()); - - // and in a liveness check, one outstanding - ApplicationLivenessInformation liveness = appState - .getApplicationLivenessInformation(); - assertEquals(1, liveness.requestsOutstanding); - assertFalse(liveness.allRequestsSatisfied); - - appState.refreshClusterStatus(); - Application application = appState.getClusterStatus(); - // TODO cluster status returns liveness info -// assertEquals(1, cd.liveness.requestsOutstanding); - - } - - - //@Test - public void testFlexDownTwice() throws Throwable { - int r0 = 6; - int r1 = 0; - int r2 = 0; - getRole0Status().setDesired(r0); - getRole1Status().setDesired(r1); - getRole2Status().setDesired(r2); - List<RoleInstance> instances = createAndStartNodes(); - - int clusterSize = r0 + r1 + r2; - assertEquals(instances.size(), clusterSize); - LOG.info("shrinking cluster"); - r0 = 4; - getRole0Status().setDesired(r0); - List<AppState.NodeCompletionResult> completionResults = new ArrayList<>(); - instances = createStartAndStopNodes(completionResults); - assertEquals(0, instances.size()); - // assert two nodes were released - assertEquals(2, completionResults.size()); - - // no-op review - completionResults = new ArrayList<>(); - instances = createStartAndStopNodes(completionResults); - assertEquals(0, instances.size()); - // assert two nodes were released - assertEquals(0, completionResults.size()); - - - // now shrink again - getRole0Status().setDesired(1); - completionResults = new ArrayList<>(); - instances = createStartAndStopNodes(completionResults); - assertEquals(0, instances.size()); - // assert two nodes were released - assertEquals(3, completionResults.size()); - - } - - //@Test - public void testFlexNegative() throws Throwable { - int r0 = 6; - int r1 = 0; - int r2 = 0; - getRole0Status().setDesired(r0); - getRole1Status().setDesired(r1); - getRole2Status().setDesired(r2); - List<RoleInstance> instances = createAndStartNodes(); - - int clusterSize = r0 + r1 + r2; - assertEquals(instances.size(), clusterSize); - LOG.info("shrinking cluster"); - getRole0Status().setDesired(-2); - List<AppState.NodeCompletionResult> completionResults = new ArrayList<>(); - try { - createStartAndStopNodes(completionResults); - fail("expected an exception"); - } catch (TriggerClusterTeardownException e) { - } - - } - - //@Test - public void testCancelWithRequestsOutstanding() throws Throwable { - // flex cluster size before the original set were allocated - - - getRole0Status().setDesired(6); - // build the ops - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - // here the data structures exist - - // go down - getRole0Status().setDesired(3); - List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes(); - assertEquals(3, ops2.size()); - for (AbstractRMOperation op : ops2) { - assertTrue(op instanceof CancelSingleRequest); - } - - } - -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org