http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java deleted file mode 100644 index 8686479..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.CancelSingleRequest; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority; -import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole; - -/** - * Test app state RM operations. - */ -public class TestMockAppStateRMOperations extends BaseMockAppStateTest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(BaseMockAppStateTest.class); - - @Override - public String getTestName() { - return "TestMockAppStateRMOperations"; - } - - //@Test - public void testPriorityOnly() throws Throwable { - assertEquals(5, extractRole(buildPriority(5, false))); - } - - //@Test - public void testPriorityRoundTrip() throws Throwable { - assertEquals(5, extractRole(buildPriority(5, false))); - } - - //@Test - public void testPriorityRoundTripWithRequest() throws Throwable { - int priority = buildPriority(5, false); - assertEquals(5, extractRole(priority)); - } - - //@Test - public void testMockAddOp() throws Throwable { - getRole0Status().setDesired(1); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 1); - ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0); - int priority = operation.getRequest().getPriority().getPriority(); - assertEquals(extractRole(priority), getRole0Status().getKey()); - MockRMOperationHandler handler = new MockRMOperationHandler(); - handler.execute(ops); - - AbstractRMOperation op = handler.getFirstOp(); - assertTrue(op instanceof ContainerRequestOperation); - } - - /** - * Test of a flex up and down op which verifies that outstanding - * requests are cancelled first. - * <ol> - * <li>request 5 nodes, assert 5 request made</li> - * <li>allocate 1 of them</li> - * <li>flex cluster size to 3</li> - * <li>assert this generates 2 cancel requests</li> - * </ol> - */ - //@Test - public void testRequestThenCancelOps() throws Throwable { - RoleStatus role0 = getRole0Status(); - role0.setDesired(5); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 5); - // now 5 outstanding requests. - assertEquals(5, role0.getRequested()); - - // allocate one - List<AbstractRMOperation> processed = new ArrayList<>(); - processed.add(ops.get(0)); - List<ContainerId> released = new ArrayList<>(); - List<AppState.NodeCompletionResult> completionResults = new ArrayList<>(); - submitOperations(processed, released); - List<RoleInstance> instances = createAndSubmitNodes(released); - processSubmissionOperations(instances, completionResults, released); - - - // four outstanding - assertEquals(4, role0.getRequested()); - - // flex cluster to 3 - role0.setDesired(3); - ops = appState.reviewRequestAndReleaseNodes(); - - // expect two cancel operation from review - assertListLength(ops, 2); - for (AbstractRMOperation op : ops) { - assertTrue(op instanceof CancelSingleRequest); - } - - MockRMOperationHandler handler = new MockRMOperationHandler(); - handler.setAvailableToCancel(4); - handler.execute(ops); - assertEquals(2, handler.getAvailableToCancel()); - assertEquals(2, role0.getRequested()); - - // flex down one more - role0.setDesired(2); - ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 1); - for (AbstractRMOperation op : ops) { - assertTrue(op instanceof CancelSingleRequest); - } - handler.execute(ops); - assertEquals(1, handler.getAvailableToCancel()); - assertEquals(1, role0.getRequested()); - } - - //@Test - public void testCancelNoActualContainers() throws Throwable { - RoleStatus role0 = getRole0Status(); - role0.setDesired(5); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 5); - // now 5 outstanding requests. - assertEquals(5, role0.getRequested()); - role0.setDesired(0); - ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 5); - - } - - - //@Test - public void testFlexDownOutstandingRequests() throws Throwable { - // engine only has two nodes, so > 2 will be outstanding - engine = new MockYarnEngine(1, 2); - List<AbstractRMOperation> ops; - // role: desired = 2, requested = 1, actual=1 - RoleStatus role0 = getRole0Status(); - role0.setDesired(4); - createAndSubmitNodes(); - - assertEquals(2, role0.getRequested()); - assertEquals(2, role0.getRunning()); - // there are now two outstanding, two actual - // Release 3 and verify that the two - // cancellations were combined with a release - role0.setDesired(1); - assertEquals(-3, role0.getDelta()); - ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 3); - int numCancel = 0; - int numRelease = 0; - for (AbstractRMOperation op : ops) { - if (op instanceof CancelSingleRequest) { - numCancel++; - } - if (op instanceof ContainerReleaseOperation) { - numRelease++; - } - } - assertEquals(2, numCancel); - assertEquals(1, numRelease); - assertEquals(0, role0.getRequested()); - // TODO releasing? -// assertEquals(1, role0.getReleasing()); - } - - //@Test - public void testCancelAllOutstandingRequests() throws Throwable { - - // role: desired = 2, requested = 1, actual=1 - RoleStatus role0 = getRole0Status(); - role0.setDesired(2); - List<AbstractRMOperation> ops; - ops = appState.reviewRequestAndReleaseNodes(); - int count = 0; - for (AbstractRMOperation op : ops) { - if (op instanceof ContainerRequestOperation) { - count++; - } - } - assertEquals(2, count); - - // there are now two outstanding, two actual - // Release 3 and verify that the two - // cancellations were combined with a release - role0.setDesired(0); - ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(2, ops.size()); - - for (AbstractRMOperation op : ops) { - assertTrue(op instanceof CancelSingleRequest); - } - } - - - //@Test - public void testFlexUpOutstandingRequests() throws Throwable { - - List<AbstractRMOperation> ops; - // role: desired = 2, requested = 1, actual=1 - RoleStatus role0 = getRole0Status(); - role0.setDesired(2); - appState.incRunningContainers(role0); - appState.incRequestedContainers(role0); - - // flex up 2 nodes, yet expect only one node to be requested, - // as the outstanding request is taken into account - role0.setDesired(4); - appState.incRequestedContainers(role0); - - assertEquals(1, role0.getRunning()); - assertEquals(2, role0.getRequested()); - assertEquals(3, role0.getActualAndRequested()); - assertEquals(1, role0.getDelta()); - ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 1); - assertTrue(ops.get(0) instanceof ContainerRequestOperation); - assertEquals(3, role0.getRequested()); - } - - //@Test - public void testFlexUpNoSpace() throws Throwable { - // engine only has two nodes, so > 2 will be outstanding - engine = new MockYarnEngine(1, 2); - // role: desired = 2, requested = 1, actual=1 - RoleStatus role0 = getRole0Status(); - role0.setDesired(4); - createAndSubmitNodes(); - - assertEquals(2, role0.getRequested()); - assertEquals(2, role0.getRunning()); - role0.setDesired(8); - assertEquals(4, role0.getDelta()); - createAndSubmitNodes(); - assertEquals(6, role0.getRequested()); - } - - - //@Test - public void testAllocateReleaseOp() throws Throwable { - getRole0Status().setDesired(1); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0); - AMRMClient.ContainerRequest request = operation.getRequest(); - Container cont = engine.allocateContainer(request); - List<Container> allocated = new ArrayList<>(); - allocated.add(cont); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> operations = new ArrayList<>(); - appState.onContainersAllocated(allocated, assignments, operations); - - assertListLength(ops, 1); - assertListLength(assignments, 1); - ContainerAssignment assigned = assignments.get(0); - Container target = assigned.container; - assertEquals(target.getId(), cont.getId()); - int roleId = assigned.role.getPriority(); - assertEquals(roleId, extractRole(request.getPriority())); - assertEquals(assigned.role.getName(), ROLE0); - RoleInstance ri = roleInstance(assigned); - //tell the app it arrived - appState.containerStartSubmitted(target, ri); - appState.innerOnNodeManagerContainerStarted(target.getId()); - assertEquals(1, getRole0Status().getRunning()); - - //now release it by changing the role status - getRole0Status().setDesired(0); - ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 1); - - assertTrue(ops.get(0) instanceof ContainerReleaseOperation); - ContainerReleaseOperation release = (ContainerReleaseOperation) ops.get(0); - assertEquals(release.getContainerId(), cont.getId()); - } - - //@Test - public void testComplexAllocation() throws Throwable { - getRole0Status().setDesired(1); - getRole1Status().setDesired(3); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - List<Container> allocations = engine.execute(ops); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> releases = new ArrayList<>(); - appState.onContainersAllocated(allocations, assignments, releases); - // we expect four release requests here for all the allocated containers - assertListLength(releases, 4); - for (AbstractRMOperation op : releases) { - assertTrue(op instanceof CancelSingleRequest); - } - assertListLength(assignments, 4); - for (ContainerAssignment assigned : assignments) { - Container target = assigned.container; - RoleInstance ri = roleInstance(assigned); - appState.containerStartSubmitted(target, ri); - } - //insert some async operation here - for (ContainerAssignment assigned : assignments) { - Container target = assigned.container; - appState.innerOnNodeManagerContainerStarted(target.getId()); - } - assertEquals(4, engine.containerCount()); - getRole1Status().setDesired(0); - ops = appState.reviewRequestAndReleaseNodes(); - assertListLength(ops, 3); - allocations = engine.execute(ops); - assertEquals(1, engine.containerCount()); - - appState.onContainersAllocated(allocations, assignments, releases); - assertTrue(assignments.isEmpty()); - assertTrue(releases.isEmpty()); - } - - //@Test - public void testDoubleNodeManagerStartEvent() throws Throwable { - getRole0Status().setDesired(1); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - List<Container> allocations = engine.execute(ops); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> releases = new ArrayList<>(); - appState.onContainersAllocated(allocations, assignments, releases); - assertListLength(assignments, 1); - ContainerAssignment assigned = assignments.get(0); - Container target = assigned.container; - RoleInstance ri = roleInstance(assigned); - appState.containerStartSubmitted(target, ri); - RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target - .getId()); - assertEquals(ri2, ri); - //try a second time, expect an error - try { - appState.innerOnNodeManagerContainerStarted(target.getId()); - fail("Expected an exception"); - } catch (RuntimeException expected) { - // expected - } - //and non-faulter should not downgrade to a null - LOG.warn("Ignore any exception/stack trace that appears below"); - LOG.warn("==============================================================="); - RoleInstance ri3 = appState.onNodeManagerContainerStarted(target.getId()); - LOG.warn("==============================================================="); - LOG.warn("Ignore any exception/stack trace that appeared above"); - assertNull(ri3); - } - - //@Test - public void testDoubleAllocate() throws Throwable { - getRole0Status().setDesired(1); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0); - AMRMClient.ContainerRequest request = operation.getRequest(); - Container cont = engine.allocateContainer(request); - List<Container> allocated = new ArrayList<>(); - allocated.add(cont); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> operations = new ArrayList<>(); - assertEquals(0L, getRole0Status().getRunning()); - assertEquals(1L, getRole0Status().getRequested()); - appState.onContainersAllocated(allocated, assignments, operations); - - assertListLength(ops, 1); - assertListLength(assignments, 1); - ContainerAssignment assigned = assignments.get(0); - Container target = assigned.container; - assertEquals(target.getId(), cont.getId()); - int roleId = assigned.role.getPriority(); - assertEquals(roleId, extractRole(request.getPriority())); - assertEquals(assigned.role.getName(), ROLE0); - RoleInstance ri = roleInstance(assigned); - //tell the app it arrived - appState.containerStartSubmitted(target, ri); - appState.innerOnNodeManagerContainerStarted(target.getId()); - assertEquals(1L, getRole0Status().getRunning()); - assertEquals(0L, getRole0Status().getRequested()); - - // now get an extra allocation that should be released - cont = engine.allocateContainer(request); - allocated = new ArrayList<>(); - allocated.add(cont); - assignments = new ArrayList<>(); - operations = new ArrayList<>(); - appState.onContainersAllocated(allocated, assignments, operations); - - assertListLength(operations, 1); - assertTrue(operations.get(0) instanceof ContainerReleaseOperation); - ContainerReleaseOperation release = (ContainerReleaseOperation) - operations.get(0); - assertEquals(release.getContainerId(), cont.getId()); - - assertEquals(1L, getRole0Status().getRunning()); - assertEquals(0L, getRole0Status().getRequested()); - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java deleted file mode 100644 index d257248..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.slider.api.resource.Application; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockAppState; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.NodeMap; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Test that app state is rebuilt on a restart. - */ -public class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest - implements MockRoles { - - @Override - public String getTestName() { - return "TestMockAppStateRebuildOnAMRestart"; - } - - //@Test - public void testRebuild() throws Throwable { - - int r0 = 1; - int r1 = 2; - int r2 = 3; - getRole0Status().setDesired(r0); - getRole1Status().setDesired(r1); - getRole2Status().setDesired(r2); - List<RoleInstance> instances = createAndStartNodes(); - - int clusterSize = r0 + r1 + r2; - assertEquals(instances.size(), clusterSize); - - //clone the list - List<Container> containers = new ArrayList<>(); - for (RoleInstance ri : instances) { - containers.add(ri.container); - } - NodeMap nodemap = appState.getRoleHistory().cloneNodemap(); - - //and rebuild - - AppStateBindingInfo bindingInfo = buildBindingInfo(); - bindingInfo.application = factory.newApplication(r0, r1, r2) - .name(getValidTestName()); - bindingInfo.liveContainers = containers; - appState = new MockAppState(bindingInfo); - - assertEquals(appState.getLiveContainers().size(), clusterSize); - - appState.getRoleHistory().dump(); - - //check that the app state direct structures match - List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0); - List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1); - List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2); - - assertEquals(r0, r0live.size()); - assertEquals(r1, r1live.size()); - assertEquals(r2, r2live.size()); - - //now examine the role history - NodeMap newNodemap = appState.getRoleHistory().cloneNodemap(); - - for (NodeInstance nodeInstance : newNodemap.values()) { - String hostname = nodeInstance.hostname; - NodeInstance orig = nodemap.get(hostname); - assertNotNull("Null entry in original nodemap for " + hostname, orig); - - for (int i : Arrays.asList(getRole0Status().getKey(), getRole1Status() - .getKey(), getRole2Status().getKey())) { - assertEquals(nodeInstance.getActiveRoleInstances(i), orig - .getActiveRoleInstances(i)); - NodeEntry origRE = orig.getOrCreate(i); - NodeEntry newRE = nodeInstance.getOrCreate(i); - assertEquals(origRE.getLive(), newRE.getLive()); - assertEquals(0, newRE.getStarting()); - } - } - assertEquals(0, appState.reviewRequestAndReleaseNodes().size()); - - Application application = appState.getClusterStatus(); - // verify the AM restart container count was set - Long restarted = application.getNumberOfRunningContainers(); - assertNotNull(restarted); - //and that the count == 1 master + the region servers - assertEquals(restarted.longValue(), (long)containers.size()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java deleted file mode 100644 index 2eccd1b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.CancelSingleRequest; -import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.ContainerAssignment; -import org.apache.slider.server.appmaster.state.RoleHistoryUtils; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole; - -/** - * Test that the app state lets you ask for nodes, get a specific host, - * release it and then get that one back again. - */ -public class TestMockAppStateRolePlacement extends BaseMockAppStateTest - implements MockRoles { - - @Override - public String getTestName() { - return "TestMockAppStateRolePlacement"; - } - - - //@Test - public void testAllocateReleaseRealloc() throws Throwable { - getRole0Status().setDesired(1); - - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - ContainerRequestOperation operation = (ContainerRequestOperation)ops - .get(0); - AMRMClient.ContainerRequest request = operation.getRequest(); - assertTrue(request.getRelaxLocality()); - assertNull(request.getNodes()); - assertNull(request.getRacks()); - assertNotNull(request.getCapability()); - - Container allocated = engine.allocateContainer(request); - List<ContainerAssignment> assignments = new ArrayList<>(); - List<AbstractRMOperation> releaseOperations = new ArrayList<>(); - appState.onContainersAllocated(Arrays.asList((Container)allocated), - assignments, releaseOperations); - // verify the release matches the allocation - assertEquals(releaseOperations.size(), 1); - CancelSingleRequest cancelOp = (CancelSingleRequest)releaseOperations - .get(0); - assertNotNull(cancelOp.getRequest()); - assertNotNull(cancelOp.getRequest().getCapability()); - assertEquals(cancelOp.getRequest().getCapability(), allocated - .getResource()); - // now the assignment - assertEquals(assignments.size(), 1); - ContainerAssignment assigned = assignments.get(0); - Container container = assigned.container; - assertEquals(container.getId(), allocated.getId()); - int roleId = assigned.role.getPriority(); - assertEquals(roleId, extractRole(request.getPriority())); - assertEquals(assigned.role.getName(), ROLE0); - String containerHostname = RoleHistoryUtils.hostnameOf(container); - RoleInstance ri = roleInstance(assigned); - //tell the app it arrived - appState.containerStartSubmitted(container, ri); - assertNotNull(appState.onNodeManagerContainerStarted(container.getId())); - assertEquals(getRole0Status().getRunning(), 1); - ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(ops.size(), 0); - - //now it is surplus - getRole0Status().setDesired(0); - ops = appState.reviewRequestAndReleaseNodes(); - ContainerReleaseOperation release = (ContainerReleaseOperation) ops.get(0); - - assertEquals(release.getContainerId(), container.getId()); - engine.execute(ops); - assertNotNull(appState.onCompletedContainer(containerStatus(container - .getId())).roleInstance); - - //view the world - appState.getRoleHistory().dump(); - - //now ask for a new one - getRole0Status().setDesired(1); - ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(ops.size(), 1); - operation = (ContainerRequestOperation) ops.get(0); - AMRMClient.ContainerRequest request2 = operation.getRequest(); - assertNotNull(request2); - assertEquals(request2.getNodes().get(0), containerHostname); - assertFalse(request2.getRelaxLocality()); - engine.execute(ops); - - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java deleted file mode 100644 index b6c8526..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -/** - * Test that if you have >1 role, the right roles are chosen for release. - */ -public class TestMockAppStateRoleRelease extends BaseMockAppStateTest - implements MockRoles { - - @Override - public String getTestName() { - return "TestMockAppStateRoleRelease"; - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4); - } - - //@Test - public void testAllocateReleaseRealloc() throws Throwable { - /** - * Allocate to all nodes - */ - getRole0Status().setDesired(6); - getRole1Status().setDesired(5); - getRole2Status().setDesired(4); - List<RoleInstance> instances = createAndStartNodes(); - assertEquals(instances.size(), 15); - - //now it is surplus - getRole0Status().setDesired(0); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - - List<ContainerId> released = new ArrayList<>(); - engine.execute(ops, released); - List<ContainerId> ids = extractContainerIds(instances, ROLE0); - for (ContainerId cid : released) { - assertNotNull(appState.onCompletedContainer(containerStatus(cid)) - .roleInstance); - assertTrue(ids.contains(cid)); - } - - //view the world - appState.getRoleHistory().dump(); - - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java deleted file mode 100644 index 8dae8e7..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.Resource; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.state.AppStateBindingInfo; -import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; - -/** - * Test that if you have more than one role, the right roles are chosen for - * release. - */ -public class TestMockAppStateUniqueNames extends BaseMockAppStateTest - implements MockRoles { - - @Override - public String getTestName() { - return "TestMockAppStateUniqueNames"; - } - - /** - * Small cluster with multiple containers per node, - * to guarantee many container allocations on each node. - * @return - */ - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(4, 4); - } - - @Override - public AppStateBindingInfo buildBindingInfo() throws IOException { - AppStateBindingInfo bindingInfo = super.buildBindingInfo(); - bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); - return bindingInfo; - } - - @Override - public Application buildApplication() { - Application application = super.buildApplication(); - - Component component = new Component().name("group1").numberOfContainers(2L) - .resource(new Resource().memory("1024").cpus(2)) - .uniqueComponentSupport(true); - - application.getComponents().add(component); - return application; - } - - public static Map<String, RoleInstance> organize(List<RoleInstance> - instances) { - Map<String, RoleInstance> map = new TreeMap<>(); - for (RoleInstance instance : instances) { - assertFalse("Multiple role instances for unique name " + instance - .compInstanceName, map.containsKey(instance.compInstanceName)); - System.out.println("Adding to map " + instance.compInstanceName + " for" + - instance.role); - map.put(instance.compInstanceName, instance); - } - return map; - } - - public static void verifyInstances(List<RoleInstance> instances, String - group, String... roles) { - assertEquals(roles.length, instances.size()); - Map<String, RoleInstance> map = organize(instances); - int i = 0; - for (Entry<String, RoleInstance> entry : map.entrySet()) { - assertEquals(roles[i], entry.getKey()); - RoleInstance instance = entry.getValue(); - assertEquals(roles[i], instance.compInstanceName); - assertEquals(i, instance.componentId); - assertEquals(group, instance.role); - assertEquals(group, instance.providerRole.name); - i++; - } - } - - //@Test - public void testDynamicFlexDown() throws Throwable { - createAndStartNodes(); - List<RoleInstance> instances = appState.cloneOwnedContainerList(); - verifyInstances(instances, "group1", "group10", "group11"); - - appState.updateComponents(Collections.singletonMap("group1", 0L)); - createAndStartNodes(); - instances = appState.cloneOwnedContainerList(); - assertEquals(0, instances.size()); - - RoleStatus roleStatus = appState.lookupRoleStatus("group1"); - assertEquals(0, roleStatus.getDesired()); - assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize()); - assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores()); - - // now flex back up - appState.updateComponents(Collections.singletonMap("group1", 3L)); - createAndStartNodes(); - instances = appState.cloneOwnedContainerList(); - verifyInstances(instances, "group1", "group10", "group11", "group12"); - } - - //@Test - public void testDynamicFlexUp() throws Throwable { - createAndStartNodes(); - List<RoleInstance> instances = appState.cloneOwnedContainerList(); - verifyInstances(instances, "group1", "group10", "group11"); - - appState.updateComponents(Collections.singletonMap("group1", 3L)); - createAndStartNodes(); - instances = appState.cloneOwnedContainerList(); - verifyInstances(instances, "group1", "group10", "group11", "group12"); - - RoleStatus group1 = appState.lookupRoleStatus("group1"); - assertEquals(3, group1.getDesired()); - assertEquals(1024L, group1.getResourceRequirements().getMemorySize()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java deleted file mode 100644 index 8bf2742..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockAppState; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; - -import java.util.Collections; -import java.util.List; - -/** - * Test the container resource allocation logic. - */ -public class TestMockContainerResourceAllocations extends BaseMockAppStateTest { - - @Override - public Application buildApplication() { - return factory.newApplication(1, 0, 0).name(getValidTestName()); - } - - //@Test - public void testNormalAllocations() throws Throwable { - Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0); - role0.resource(new org.apache.slider.api.resource.Resource().memory("512") - .cpus(2)); - // hack - because role0 is created before the test run - RoleStatus role0Status = - appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id); - role0Status.setResourceRequirements( - appState.buildResourceRequirements(role0Status)); - appState.updateComponents(Collections.singletonMap(role0.getName(), - role0.getNumberOfContainers())); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, ops.size()); - ContainerRequestOperation operation = (ContainerRequestOperation) ops - .get(0); - Resource requirements = operation.getRequest().getCapability(); - assertEquals(512L, requirements.getMemorySize()); - assertEquals(2, requirements.getVirtualCores()); - } - - //TODO replace with resource profile feature in yarn - //@Test - public void testMaxMemAllocations() throws Throwable { - // max core allocations no longer supported - Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0); - role0.resource(new org.apache.slider.api.resource.Resource() - .memory(ResourceKeys.YARN_RESOURCE_MAX).cpus(2)); - RoleStatus role0Status = - appState.getRoleStatusMap().get(appState.getRoleMap().get(ROLE0).id); - role0Status.setResourceRequirements( - appState.buildResourceRequirements(role0Status)); - appState.updateComponents(Collections.singletonMap(role0.getName(), - role0.getNumberOfContainers())); - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(1, ops.size()); - ContainerRequestOperation operation = (ContainerRequestOperation) ops - .get(0); - Resource requirements = operation.getRequest().getCapability(); - assertEquals(MockAppState.RM_MAX_RAM, requirements.getMemorySize()); - assertEquals(2, requirements.getVirtualCores()); - } - - //@Test - public void testMaxDefaultAllocations() throws Throwable { - List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); - assertEquals(ops.size(), 1); - ContainerRequestOperation operation = (ContainerRequestOperation) ops - .get(0); - Resource requirements = operation.getRequest().getCapability(); - assertEquals(ResourceKeys.DEF_YARN_MEMORY, requirements.getMemorySize()); - assertEquals(ResourceKeys.DEF_YARN_CORES, requirements.getVirtualCores()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java deleted file mode 100644 index 453b14c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.slider.server.appmaster.model.mock.MockFactory; -import org.apache.slider.server.appmaster.model.mock.MockRoles; -import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.AppState; -import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * Test Anti-affine placement. - */ -public class TestMockLabelledAAPlacement extends BaseMockAppStateAATest - implements MockRoles { - private static final Logger LOG = - LoggerFactory.getLogger(TestMockLabelledAAPlacement.class); - - private static final int NODES = 3; - private static final int GPU_NODES = 2; - private static final String HOST0 = "00000000"; - private static final String HOST1 = "00000001"; - - @Override - public void setup() throws Exception { - super.setup(); - - updateNodes(MockFactory.INSTANCE.newNodeReport(HOST0, NodeState.RUNNING, - LABEL_GPU)); - updateNodes(MockFactory.INSTANCE.newNodeReport(HOST1, NodeState.RUNNING, - LABEL_GPU)); - } - - @Override - public MockYarnEngine createYarnEngine() { - return new MockYarnEngine(NODES, 8); - } - - void assertAllContainersAA() { - assertAllContainersAA(getGpuRole().getKey()); - } - - /** - * - * @throws Throwable - */ - //@Test - public void testAskForTooMany() throws Throwable { - RoleStatus gpuRole = getGpuRole(); - - describe("Ask for 1 more than the no of available nodes;" + - " expect the final request to be unsatisfied until the cluster " + - "changes size"); - //more than expected - int size = GPU_NODES; - gpuRole.setDesired(size + 1); - - List<AbstractRMOperation > operations = appState - .reviewRequestAndReleaseNodes(); - assertTrue(gpuRole.isAARequestOutstanding()); - - assertEquals(gpuRole.getAAPending(), size); - for (int i = 0; i < size; i++) { - String iter = "Iteration " + i + " role = " + getAaRole(); - describe(iter); - List<AbstractRMOperation > operationsOut = new ArrayList<>(); - - List<RoleInstance> roleInstances = submitOperations(operations, - EMPTY_ID_LIST, operationsOut); - // one instance per request - assertEquals(1, roleInstances.size()); - appState.onNodeManagerContainerStarted(roleInstances.get(0) - .getContainerId()); - assertAllContainersAA(); - // there should be none left - LOG.debug(nodeInformationSnapshotAsString()); - operations = operationsOut; - if (i + 1 < size) { - assertEquals(2, operations.size()); - } else { - assertEquals(1, operations.size()); - } - } - // expect an outstanding AA request to be unsatisfied - assertTrue(gpuRole.getRunning() < gpuRole.getDesired()); - assertEquals(0, gpuRole.getRequested()); - assertFalse(gpuRole.isAARequestOutstanding()); - List<Container> allocatedContainers = engine.execute(operations, - EMPTY_ID_LIST); - assertEquals(0, allocatedContainers.size()); - // in a review now, no more requests can be generated, as there is no - // space for AA placements, even though there is cluster capacity - assertEquals(0, appState.reviewRequestAndReleaseNodes().size()); - - // switch node 2 into being labelled - NodeUpdatedOutcome outcome = updateNodes(MockFactory.INSTANCE. - newNodeReport("00000002", NodeState.RUNNING, "gpu")); - - assertEquals(NODES, cloneNodemap().size()); - assertTrue(outcome.clusterChanged); - // no active calls to empty - assertTrue(outcome.operations.isEmpty()); - assertEquals(1, appState.reviewRequestAndReleaseNodes().size()); - } - - protected AppState.NodeUpdatedOutcome addNewNode() { - return updateNodes(MockFactory.INSTANCE.newNodeReport("00000004", - NodeState.RUNNING, "gpu")); - } - - //@Test - public void testClusterSizeChangesDuringRequestSequence() throws Throwable { - RoleStatus gpuRole = getGpuRole(); - describe("Change the cluster size where the cluster size changes during " + - "a test sequence."); - gpuRole.setDesired(GPU_NODES + 1); - List<AbstractRMOperation> operations = appState - .reviewRequestAndReleaseNodes(); - assertTrue(gpuRole.isAARequestOutstanding()); - assertEquals(GPU_NODES, gpuRole.getAAPending()); - NodeUpdatedOutcome outcome = addNewNode(); - assertTrue(outcome.clusterChanged); - // one call to cancel - assertEquals(1, outcome.operations.size()); - // and on a review, one more to rebuild - assertEquals(1, appState.reviewRequestAndReleaseNodes().size()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java deleted file mode 100644 index 65d1d86..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate; - -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.slider.server.appmaster.state.ContainerPriority; -import org.apache.slider.server.appmaster.state.OutstandingRequest; -import org.apache.slider.utils.SliderTestBase; -import org.junit.Test; - -/** - * Test outstanding request validation. - */ -public class TestOutstandingRequestValidation extends SliderTestBase { - - private static final String[] H1 = hosts("one"); - - //@Test - public void testRelaxedNohostsOrLabels() throws Throwable { - createAndValidate(null, null, true); - } - - //@Test - public void testRelaxedLabels() throws Throwable { - createAndValidate(null, "gpu", true); - } - - //@Test - public void testNonRelaxedLabels() throws Throwable { - expectCreationFailure(null, "gpu", false); - } - - //@Test - public void testRelaxedHostNoLabel() throws Throwable { - createAndValidate(H1, "", true); - } - - /** - * Use varargs for simple list to array conversion. - * @param hostnames host names - * @return - */ - public static String[] hosts(String...hostnames) { - return hostnames; - } - - void expectCreationFailure( - String[] hosts, - String labels, - boolean relaxLocality) { - try { - ContainerRequest result = createAndValidate(hosts, labels, relaxLocality); - fail("Expected an exception, got " + result); - } catch (IllegalArgumentException expected) { - assertTrue(expected.toString() - .contains("Can't turn off locality relaxation on a request with no " + - "location constraints")); - } - } - - AMRMClient.ContainerRequest createAndValidate( - String[] hosts, - String labels, - boolean relaxLocality) { - int cores = 1; - int memory = 64; - int p = 1; - Priority pri = ContainerPriority.createPriority(p, !relaxLocality); - ContainerRequest issuedRequest = - newRequest(pri, hosts, labels, relaxLocality); - OutstandingRequest.validateContainerRequest(issuedRequest, p, ""); - return issuedRequest; - } - - AMRMClient.ContainerRequest newRequest( - Priority pri, - String[] hosts, - String labels, - boolean relaxLocality) { - int cores = 1; - int memory = 64; - Resource resource = Resource.newInstance(memory, cores); - return new AMRMClient.ContainerRequest(resource, - hosts, - null, - pri, - relaxLocality, - labels); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java deleted file mode 100644 index 0cdb952..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history; - -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.slider.api.proto.Messages; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.api.types.RestTypeMarshalling; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.server.appmaster.model.mock.MockFactory; -import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.NodeMap; -import org.apache.slider.server.appmaster.state.RoleHistory; -import org.apache.slider.utils.SliderTestBase; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Test anti-affine placement. - */ -public class TestRoleHistoryAA extends SliderTestBase { - private static final Logger LOG = - LoggerFactory.getLogger(TestRoleHistoryAA.class); - - private List<String> hostnames = Arrays.asList("1", "2", "3"); - private NodeMap nodeMap, gpuNodeMap; - private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); - - public TestRoleHistoryAA() throws BadConfigException { - } - - @Override - public void setup() throws Exception { - super.setup(); - nodeMap = createNodeMap(hostnames, NodeState.RUNNING, ""); - gpuNodeMap = createNodeMap(hostnames, NodeState.RUNNING, "GPU"); - } - - //@Test - public void testFindNodesInFullCluster() throws Throwable { - // all three will surface at first - verifyResultSize(3, nodeMap.findAllNodesForRole(1, "")); - } - - //@Test - public void testFindNodesInUnhealthyCluster() throws Throwable { - // all three will surface at first - markNodeOneUnhealthy(); - verifyResultSize(2, nodeMap.findAllNodesForRole(1, "")); - } - - public boolean markNodeOneUnhealthy() { - return setNodeState(nodeMap.get("1"), NodeState.UNHEALTHY); - } - - protected boolean setNodeState(NodeInstance node, NodeState state) { - return node.updateNode(MockFactory.INSTANCE.newNodeReport(node.hostname, - state, "")); - } - - //@Test - public void testFindNoNodesWrongLabel() throws Throwable { - // all three will surface at first - verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU")); - } - - //@Test - public void testFindSomeNodesSomeLabel() throws Throwable { - // all three will surface at first - update(nodeMap, - Arrays.asList(MockFactory.INSTANCE.newNodeReport("1", NodeState - .RUNNING, "GPU"))); - List<NodeInstance> gpuNodes = nodeMap.findAllNodesForRole(1, "GPU"); - verifyResultSize(1, gpuNodes); - NodeInstance instance = gpuNodes.get(0); - instance.getOrCreate(1).onStarting(); - assertFalse(instance.canHost(1, "GPU")); - assertFalse(instance.canHost(1, "")); - verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU")); - - } - - //@Test - public void testFindNoNodesRightLabel() throws Throwable { - // all three will surface at first - verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU")); - } - - //@Test - public void testFindNoNodesNoLabel() throws Throwable { - // all three will surface at first - verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "")); - } - - //@Test - public void testFindNoNodesClusterRequested() throws Throwable { - // all three will surface at first - for (NodeInstance ni : nodeMap.values()) { - ni.getOrCreate(1).request(); - } - assertNoAvailableNodes(1); - } - - //@Test - public void testFindNoNodesClusterBusy() throws Throwable { - // all three will surface at first - for (NodeInstance ni : nodeMap.values()) { - ni.getOrCreate(1).request(); - } - assertNoAvailableNodes(1); - } - - /** - * Tag all nodes as starting, then walk one through a bit - * more of its lifecycle. - */ - //@Test - public void testFindNoNodesLifecycle() throws Throwable { - // all three will surface at first - for (NodeInstance ni : nodeMap.values()) { - ni.getOrCreate(1).onStarting(); - } - assertNoAvailableNodes(1); - - // walk one of the nodes through the lifecycle - NodeInstance node1 = nodeMap.get("1"); - assertFalse(node1.canHost(1, "")); - node1.get(1).onStartCompleted(); - assertFalse(node1.canHost(1, "")); - assertNoAvailableNodes(1); - node1.get(1).release(); - assertTrue(node1.canHost(1, "")); - List<NodeInstance> list2 = - verifyResultSize(1, nodeMap.findAllNodesForRole(1, "")); - assertEquals(list2.get(0).hostname, "1"); - - // now tag that node as unhealthy and expect it to go away - markNodeOneUnhealthy(); - assertNoAvailableNodes(1); - } - - //@Test - public void testRolesIndependent() throws Throwable { - NodeInstance node1 = nodeMap.get("1"); - NodeEntry role1 = node1.getOrCreate(1); - NodeEntry role2 = node1.getOrCreate(2); - for (NodeInstance ni : nodeMap.values()) { - ni.updateNode(MockFactory.INSTANCE.newNodeReport("0", NodeState - .UNHEALTHY, "")); - } - assertNoAvailableNodes(1); - assertNoAvailableNodes(2); - assertTrue(setNodeState(node1, NodeState.RUNNING)); - // tag role 1 as busy - role1.onStarting(); - assertNoAvailableNodes(1); - - verifyResultSize(1, nodeMap.findAllNodesForRole(2, "")); - assertTrue(node1.canHost(2, "")); - } - - //@Test - public void testNodeEntryAvailablity() throws Throwable { - NodeEntry entry = new NodeEntry(1); - assertTrue(entry.isAvailable()); - entry.onStarting(); - assertFalse(entry.isAvailable()); - entry.onStartCompleted(); - assertFalse(entry.isAvailable()); - entry.release(); - assertTrue(entry.isAvailable()); - entry.onStarting(); - assertFalse(entry.isAvailable()); - entry.onStartFailed(); - assertTrue(entry.isAvailable()); - } - - //@Test - public void testNodeInstanceSerialization() throws Throwable { - MockRoleHistory rh2 = new MockRoleHistory(new ArrayList<>()); - rh2.getOrCreateNodeInstance("localhost"); - NodeInstance instance = rh2.getOrCreateNodeInstance("localhost"); - instance.getOrCreate(1).onStartCompleted(); - Map<Integer, String> naming = Collections.singletonMap(1, "manager"); - NodeInformation ni = instance.serialize(naming); - assertEquals(1, ni.entries.get("manager").live); - NodeInformation ni2 = rh2.getNodeInformation("localhost", naming); - assertEquals(1, ni2.entries.get("manager").live); - Map<String, NodeInformation> info = rh2.getNodeInformationSnapshot(naming); - assertEquals(1, info.get("localhost").entries.get("manager").live); - NodeInformationList nil = new NodeInformationList(info.values()); - assertEquals(1, nil.get(0).entries.get("manager").live); - - Messages.NodeInformationProto nodeInformationProto = - RestTypeMarshalling.marshall(ni); - Messages.NodeEntryInformationProto entryProto = nodeInformationProto - .getEntries(0); - assertNotNull(entryProto); - assertEquals(1, entryProto.getPriority()); - NodeInformation unmarshalled = - RestTypeMarshalling.unmarshall(nodeInformationProto); - assertEquals(unmarshalled.hostname, ni.hostname); - assertTrue(unmarshalled.entries.keySet().containsAll(ni.entries.keySet())); - - } - - //@Test - public void testBuildRolenames() throws Throwable { - - } - public List<NodeInstance> assertNoAvailableNodes(int role) { - String label = ""; - return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label)); - } - - List<NodeInstance> verifyResultSize(int size, List<NodeInstance> list) { - if (list.size() != size) { - for (NodeInstance ni : list) { - LOG.error(ni.toFullString()); - } - } - assertEquals(size, list.size()); - return list; - } - - NodeMap createNodeMap(List<NodeReport> nodeReports) - throws BadConfigException { - NodeMap newNodeMap = new NodeMap(1); - update(newNodeMap, nodeReports); - return newNodeMap; - } - - protected boolean update(NodeMap nm, List<NodeReport> nodeReports) { - return nm.buildOrUpdate(nodeReports); - } - - NodeMap createNodeMap(List<String> hosts, NodeState state, - String label) throws BadConfigException { - return createNodeMap(MockFactory.INSTANCE.createNodeReports(hosts, state, - label)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java deleted file mode 100644 index bbe95b9..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; -import org.apache.slider.server.appmaster.model.mock.MockContainer; -import org.apache.slider.server.appmaster.model.mock.MockNodeId; -import org.apache.slider.server.appmaster.state.ContainerOutcome; -import org.apache.slider.server.appmaster.state.ContainerPriority; -import org.apache.slider.server.appmaster.state.NodeEntry; -import org.apache.slider.server.appmaster.state.NodeInstance; -import org.apache.slider.server.appmaster.state.NodeMap; -import org.apache.slider.server.appmaster.state.RoleHistory; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Test container events at the role history level -one below - * the App State. - */ -public class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { - private static final Logger LOG = - LoggerFactory.getLogger(TestRoleHistoryContainerEvents.class); - - @Override - public String getTestName() { - return "TestRoleHistoryContainerEvents"; - } - - private NodeInstance age1Active4; - private NodeInstance age2Active2; - private NodeInstance age3Active0; - private NodeInstance age4Active1; - private NodeInstance age2Active0; - - private RoleHistory roleHistory; - - private Resource resource; - - AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) { - return roleHistory.requestContainerForRole(roleStatus).getIssuedRequest(); - } - - @Override - public void setup() throws Exception { - super.setup(); - - age1Active4 = nodeInstance(1, 4, 0, 0); - age2Active2 = nodeInstance(2, 2, 0, 1); - age3Active0 = nodeInstance(3, 0, 0, 0); - age4Active1 = nodeInstance(4, 1, 0, 0); - age2Active0 = nodeInstance(2, 0, 0, 0); - - roleHistory = appState.getRoleHistory(); - roleHistory.insert(Arrays.asList(age2Active2, age2Active0, - age4Active1, age1Active4, age3Active0)); - roleHistory.buildRecentNodeLists(); - resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES, - ResourceKeys.DEF_YARN_MEMORY); - } - - //@Test - public void testFindAndCreate() throws Throwable { - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - List<String> requestNodes = request.getNodes(); - assertNotNull(requestNodes); - assertEquals(1, requestNodes.size()); - String hostname = requestNodes.get(0); - assertEquals(hostname, age3Active0.hostname); - - //build a container - MockContainer container = factory.newContainer(); - container.setNodeId(new MockNodeId(hostname, 0)); - container.setPriority(request.getPriority()); - roleHistory.onContainerAssigned(container); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - assertEquals(1, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - RoleInstance ri = new RoleInstance(container); - //start it - roleHistory.onContainerStartSubmitted(container, ri); - //later, declare that it started - roleHistory.onContainerStarted(container); - assertEquals(0, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - assertEquals(1, roleEntry.getActive()); - assertEquals(1, roleEntry.getLive()); - } - - //@Test - public void testCreateAndRelease() throws Throwable { - RoleStatus roleStatus = getRole1Status(); - - //verify it is empty - assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty()); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - assertNull(request.getNodes()); - - //pick an idle host - String hostname = age3Active0.hostname; - - //build a container - MockContainer container = factory.newContainer(new MockNodeId(hostname, - 0), request.getPriority()); - roleHistory.onContainerAssigned(container); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - assertEquals(1, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - RoleInstance ri = new RoleInstance(container); - //start it - roleHistory.onContainerStartSubmitted(container, ri); - //later, declare that it started - roleHistory.onContainerStarted(container); - assertEquals(0, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - assertEquals(1, roleEntry.getActive()); - assertEquals(1, roleEntry.getLive()); - - // now pick that instance to destroy - List<NodeInstance> activeNodes = roleHistory.listActiveNodes(roleStatus - .getKey()); - - - assertEquals(1, activeNodes.size()); - NodeInstance target = activeNodes.get(0); - assertEquals(target, allocated); - roleHistory.onContainerReleaseSubmitted(container); - assertEquals(1, roleEntry.getReleasing()); - assertEquals(1, roleEntry.getLive()); - assertEquals(0, roleEntry.getActive()); - - // release completed - roleHistory.onReleaseCompleted(container); - assertEquals(0, roleEntry.getReleasing()); - assertEquals(0, roleEntry.getLive()); - assertEquals(0, roleEntry.getActive()); - - // verify it is empty - assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty()); - - // ask for a container and expect to get the recently released one - AMRMClient.ContainerRequest request2 = - requestContainer(roleStatus); - - List<String> nodes2 = request2.getNodes(); - assertNotNull(nodes2); - String hostname2 = nodes2.get(0); - - //pick an idle host - assertEquals(hostname2, age3Active0.hostname); - } - - - //@Test - public void testStartWithoutWarning() throws Throwable { - //pick an idle host - String hostname = age3Active0.hostname; - //build a container - MockContainer container = factory.newContainer( - new MockNodeId(hostname, 0), - ContainerPriority.createPriority(getRole0Status().getKey(), false)); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(getRole0Status().getKey()); - - //tell RH that it started - roleHistory.onContainerStarted(container); - assertEquals(0, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - assertEquals(1, roleEntry.getActive()); - assertEquals(1, roleEntry.getLive()); - } - - //@Test - public void testStartFailed() throws Throwable { - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - LOG.info("req {}", request); - LOG.info("{}", request.getNodes()); - String hostname = request.getNodes().get(0); - assertEquals(hostname, age3Active0.hostname); - - //build a container - MockContainer container = factory.newContainer(new MockNodeId(hostname, - 0), request.getPriority()); - roleHistory.onContainerAssigned(container); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - assertEquals(1, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - RoleInstance ri = new RoleInstance(container); - //start it - roleHistory.onContainerStartSubmitted(container, ri); - //later, declare that it failed on startup - assertFalse(roleHistory.onNodeManagerContainerStartFailed(container)); - assertEquals(0, roleEntry.getStarting()); - assertEquals(1, roleEntry.getStartFailed()); - assertEquals(1, roleEntry.getFailed()); - assertTrue(roleEntry.isAvailable()); - assertEquals(0, roleEntry.getActive()); - assertEquals(0, roleEntry.getLive()); - } - - //@Test - public void testStartFailedWithoutWarning() throws Throwable { - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - String hostname = request.getNodes().get(0); - assertEquals(hostname, age3Active0.hostname); - - //build a container - MockContainer container = factory.newContainer(); - container.setNodeId(new MockNodeId(hostname, 0)); - container.setPriority(request.getPriority()); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - - assertFalse(roleHistory.onNodeManagerContainerStartFailed(container)); - assertEquals(0, roleEntry.getStarting()); - assertEquals(1, roleEntry.getStartFailed()); - assertEquals(1, roleEntry.getFailed()); - assertTrue(roleEntry.isAvailable()); - assertEquals(0, roleEntry.getActive()); - assertEquals(0, roleEntry.getLive()); - } - - //@Test - public void testContainerFailed() throws Throwable { - describe("fail a container without declaring it as starting"); - - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - String hostname = request.getNodes().get(0); - assertEquals(hostname, age3Active0.hostname); - - //build a container - MockContainer container = factory.newContainer(); - container.setNodeId(new MockNodeId(hostname, 0)); - container.setPriority(request.getPriority()); - roleHistory.onContainerAssigned(container); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - assertEquals(1, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - RoleInstance ri = new RoleInstance(container); - //start it - roleHistory.onContainerStartSubmitted(container, ri); - roleHistory.onContainerStarted(container); - - //later, declare that it failed - roleHistory.onFailedContainer( - container, - false, - ContainerOutcome.Failed); - assertEquals(0, roleEntry.getStarting()); - assertTrue(roleEntry.isAvailable()); - assertEquals(0, roleEntry.getActive()); - assertEquals(0, roleEntry.getLive()); - } - - //@Test - public void testContainerFailedWithoutWarning() throws Throwable { - describe("fail a container without declaring it as starting"); - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - String hostname = request.getNodes().get(0); - assertEquals(hostname, age3Active0.hostname); - - //build a container - MockContainer container = factory.newContainer(); - container.setNodeId(new MockNodeId(hostname, 0)); - container.setPriority(request.getPriority()); - - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - assertTrue(roleEntry.isAvailable()); - roleHistory.onFailedContainer( - container, - false, - ContainerOutcome.Failed); - assertEquals(0, roleEntry.getStarting()); - assertEquals(1, roleEntry.getFailed()); - assertTrue(roleEntry.isAvailable()); - assertEquals(0, roleEntry.getActive()); - assertEquals(0, roleEntry.getLive()); - } - - //@Test - public void testAllocationListPrep() throws Throwable { - describe("test prepareAllocationList"); - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - requestContainer(roleStatus); - - String hostname = request.getNodes().get(0); - assertEquals(hostname, age3Active0.hostname); - - MockContainer container1 = factory.newContainer(); - container1.setNodeId(new MockNodeId(hostname, 0)); - container1.setPriority(Priority.newInstance(getRole0Status().getKey())); - - MockContainer container2 = factory.newContainer(); - container2.setNodeId(new MockNodeId(hostname, 0)); - container2.setPriority(Priority.newInstance(getRole1Status().getKey())); - - // put containers in List with role == 1 first - List<Container> containers = Arrays.asList((Container) container2, - (Container) container1); - List<Container> sortedContainers = roleHistory.prepareAllocationList( - containers); - - // verify that the first container has role == 0 after sorting - MockContainer c1 = (MockContainer) sortedContainers.get(0); - assertEquals(getRole0Status().getKey(), c1.getPriority().getPriority()); - MockContainer c2 = (MockContainer) sortedContainers.get(1); - assertEquals(getRole1Status().getKey(), c2.getPriority().getPriority()); - } - - //@Test - public void testNodeUpdated() throws Throwable { - describe("fail a node"); - - RoleStatus roleStatus = getRole0Status(); - - AMRMClient.ContainerRequest request = - roleHistory.requestContainerForRole(roleStatus).getIssuedRequest(); - - String hostname = request.getNodes().get(0); - assertEquals(age3Active0.hostname, hostname); - - // build a container - MockContainer container = factory.newContainer(new MockNodeId(hostname, - 0), request.getPriority()); - - roleHistory.onContainerAssigned(container); - - NodeMap nodemap = roleHistory.cloneNodemap(); - NodeInstance allocated = nodemap.get(hostname); - NodeEntry roleEntry = allocated.get(roleStatus.getKey()); - assertEquals(1, roleEntry.getStarting()); - assertFalse(roleEntry.isAvailable()); - RoleInstance ri = new RoleInstance(container); - // start it - roleHistory.onContainerStartSubmitted(container, ri); - roleHistory.onContainerStarted(container); - - int startSize = nodemap.size(); - - // now send a list of updated (failed) nodes event - List<NodeReport> nodesUpdated = new ArrayList<>(); - NodeReport nodeReport = NodeReport.newInstance( - NodeId.newInstance(hostname, 0), - NodeState.LOST, - null, null, null, null, 1, null, 0); - nodesUpdated.add(nodeReport); - roleHistory.onNodesUpdated(nodesUpdated); - - nodemap = roleHistory.cloneNodemap(); - int endSize = nodemap.size(); - // as even unused nodes are added to the list, we expect the map size to - // be >1 - assertTrue(startSize <= endSize); - assertNotNull(nodemap.get(hostname)); - assertFalse(nodemap.get(hostname).isOnline()); - - // add a failure of a node we've never head of - String newhost = "newhost"; - nodesUpdated = Arrays.asList( - NodeReport.newInstance( - NodeId.newInstance(newhost, 0), - NodeState.LOST, - null, null, null, null, 1, null, 0) - ); - roleHistory.onNodesUpdated(nodesUpdated); - - NodeMap nodemap2 = roleHistory.cloneNodemap(); - assertNotNull(nodemap2.get(newhost)); - assertFalse(nodemap2.get(newhost).isOnline()); - - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
