http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java new file mode 100644 index 0000000..7d8f5a7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.providers.PlacementPolicy; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockAppState; +import org.apache.slider.server.appmaster.model.mock.MockContainer; +import org.apache.slider.server.appmaster.model.mock.MockNodeId; +import org.apache.slider.server.appmaster.model.mock.MockPriority; +import org.apache.slider.server.appmaster.model.mock.MockResource; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.CancelSingleRequest; +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome; +import org.apache.slider.server.appmaster.state.ContainerAllocationResults; +import org.apache.slider.server.appmaster.state.ContainerPriority; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.OutstandingRequest; +import org.apache.slider.server.appmaster.state.OutstandingRequestTracker; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Test outstanding request tracker. + */ +public class TestRoleHistoryOutstandingRequestTracker extends + BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryOutstandingRequestTracker.class); + + public static final String WORKERS_LABEL = "workers"; + private NodeInstance host1 = new NodeInstance("host1", 3); + private NodeInstance host2 = new NodeInstance("host2", 3); + private MockResource resource = factory.newResource(48, 1); + + private OutstandingRequestTracker tracker = new OutstandingRequestTracker(); + + public static final String WORKER = "worker"; + + @Override + public Application buildApplication() { + Application application = super.buildApplication(); + Component component = new Component().name("worker").numberOfContainers(0L); + component.getConfiguration().setProperty(ResourceKeys.YARN_LABEL_EXPRESSION, + WORKERS_LABEL); + application.getComponents().add(component); + return application; + } + + @Test + public void testAddRetrieveEntry() throws Throwable { + OutstandingRequest request = tracker.newRequest(host1, 0); + assertEquals(tracker.lookupPlacedRequest(0, "host1"), request); + assertEquals(tracker.removePlacedRequest(request), request); + assertNull(tracker.lookupPlacedRequest(0, "host1")); + } + + @Test + public void testAddCompleteEntry() throws Throwable { + OutstandingRequest req1 = tracker.newRequest(host1, 0); + req1.buildContainerRequest(resource, getRole0Status(), 0); + + tracker.newRequest(host2, 0).buildContainerRequest(resource, + getRole0Status(), 0); + tracker.newRequest(host1, 1).buildContainerRequest(resource, + getRole0Status(), 0); + + ContainerAllocationResults allocation = tracker.onContainerAllocated(1, + "host1", null); + assertEquals(allocation.outcome, ContainerAllocationOutcome.Placed); + assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest); + + assertNull(tracker.lookupPlacedRequest(1, "host1")); + assertNotNull(tracker.lookupPlacedRequest(0, "host1")); + } + + @Test + public void testResetOpenRequests() throws Throwable { + OutstandingRequest req1 = tracker.newRequest(null, 0); + assertFalse(req1.isLocated()); + tracker.newRequest(host1, 0); + List<OutstandingRequest> openRequests = tracker.listOpenRequests(); + assertEquals(1, openRequests.size()); + tracker.resetOutstandingRequests(0); + assertTrue(tracker.listOpenRequests().isEmpty()); + assertTrue(tracker.listPlacedRequests().isEmpty()); + } + + @Test + public void testRemoveOpenRequestUnissued() throws Throwable { + OutstandingRequest req1 = tracker.newRequest(null, 0); + req1.buildContainerRequest(resource, getRole0Status(), 0); + assertEquals(1, tracker.listOpenRequests().size()); + MockContainer c1 = factory.newContainer(null, new MockPriority(0)); + c1.setResource(resource); + + ContainerAllocationResults allocation = + tracker.onContainerAllocated(0, "host1", c1); + ContainerAllocationOutcome outcome = allocation.outcome; + assertEquals(outcome, ContainerAllocationOutcome.Unallocated); + assertTrue(allocation.operations.isEmpty()); + assertEquals(1, tracker.listOpenRequests().size()); + } + + @Test + public void testIssuedOpenRequest() throws Throwable { + OutstandingRequest req1 = tracker.newRequest(null, 0); + req1.buildContainerRequest(resource, getRole0Status(), 0); + assertEquals(1, tracker.listOpenRequests().size()); + + int pri = ContainerPriority.buildPriority(0, false); + assertTrue(pri > 0); + MockNodeId nodeId = factory.newNodeId("hostname-1"); + MockContainer c1 = factory.newContainer(nodeId, new MockPriority(pri)); + + c1.setResource(resource); + + ContainerRequest issued = req1.getIssuedRequest(); + assertEquals(issued.getCapability(), resource); + assertEquals(issued.getPriority().getPriority(), c1.getPriority() + .getPriority()); + assertTrue(req1.resourceRequirementsMatch(resource)); + + ContainerAllocationResults allocation = + tracker.onContainerAllocated(0, nodeId.getHost(), c1); + assertEquals(0, tracker.listOpenRequests().size()); + assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest); + + assertEquals(allocation.outcome, ContainerAllocationOutcome.Open); + assertEquals(allocation.origin, req1); + } + + @Test + public void testResetEntries() throws Throwable { + tracker.newRequest(host1, 0); + tracker.newRequest(host2, 0); + tracker.newRequest(host1, 1); + List<NodeInstance> canceled = tracker.resetOutstandingRequests(0); + assertEquals(2, canceled.size()); + assertTrue(canceled.contains(host1)); + assertTrue(canceled.contains(host2)); + assertNotNull(tracker.lookupPlacedRequest(1, "host1")); + assertNull(tracker.lookupPlacedRequest(0, "host1")); + canceled = tracker.resetOutstandingRequests(0); + assertEquals(0, canceled.size()); + assertEquals(1, tracker.resetOutstandingRequests(1).size()); + } + + @Test + public void testEscalation() throws Throwable { + // first request: default placement + assertEquals(getRole0Status().getPlacementPolicy(), PlacementPolicy + .DEFAULT); + Resource res0 = newResource(getRole0Status()); + OutstandingRequest outstanding0 = tracker.newRequest(host1, + getRole0Status().getKey()); + ContainerRequest initialRequest = + outstanding0.buildContainerRequest(res0, getRole0Status(), 0); + assertNotNull(outstanding0.getIssuedRequest()); + assertTrue(outstanding0.isLocated()); + assertFalse(outstanding0.isEscalated()); + assertFalse(initialRequest.getRelaxLocality()); + assertEquals(1, tracker.listPlacedRequests().size()); + + // second. This one doesn't get launched. This is to verify that the + // escalation process skips entries which are in the list but have not + // been issued, which can be a race condition between request issuance & + // escalation. + // (not one observed outside test authoring, but retained for completeness) + Resource res2 = newResource(getRole2Status()); + OutstandingRequest outstanding2 = tracker.newRequest(host1, + getRole2Status().getKey()); + + // simulate some time escalation of role 1 MUST now be triggered + long interval = getRole0Status().getPlacementTimeoutSeconds() * 1000 + 500; + long now = interval; + final List<AbstractRMOperation> escalations = tracker + .escalateOutstandingRequests(now); + + assertTrue(outstanding0.isEscalated()); + assertFalse(outstanding2.isEscalated()); + + // two entries + assertEquals(2, escalations.size()); + AbstractRMOperation e1 = escalations.get(0); + assertTrue(e1 instanceof CancelSingleRequest); + final CancelSingleRequest cancel = (CancelSingleRequest) e1; + assertEquals(initialRequest, cancel.getRequest()); + AbstractRMOperation e2 = escalations.get(1); + assertTrue(e2 instanceof ContainerRequestOperation); + ContainerRequestOperation escRequest = (ContainerRequestOperation) e2; + assertTrue(escRequest.getRequest().getRelaxLocality()); + + // build that second request from an anti-affine entry + // these get placed as well + now += interval; + ContainerRequest containerReq2 = + outstanding2.buildContainerRequest(res2, getRole2Status(), now); + // escalate a little bit more + final List<AbstractRMOperation> escalations2 = tracker + .escalateOutstandingRequests(now); + // and expect no new entries + assertEquals(0, escalations2.size()); + + // go past the role2 timeout + now += getRole2Status().getPlacementTimeoutSeconds() * 1000 + 500; + // escalate a little bit more + final List<AbstractRMOperation> escalations3 = tracker + .escalateOutstandingRequests(now); + // and expect another escalation + assertEquals(2, escalations3.size()); + assertTrue(outstanding2.isEscalated()); + + // finally add a strict entry to the mix + Resource res3 = newResource(getRole1Status()); + OutstandingRequest outstanding3 = tracker.newRequest(host1, + getRole1Status().getKey()); + + final ProviderRole providerRole1 = getRole1Status().getProviderRole(); + assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT); + now += interval; + assertFalse(outstanding3.mayEscalate()); + final List<AbstractRMOperation> escalations4 = tracker + .escalateOutstandingRequests(now); + assertTrue(escalations4.isEmpty()); + + } + + /** + * If the placement does include a label, the initial request must + * <i>not</i> include it. + * The escalation request will contain the label, while + * leaving out the node list. + * retains the node list, but sets relaxLocality==true + * @throws Throwable + */ + @Test + public void testRequestLabelledPlacement() throws Throwable { + NodeInstance ni = new NodeInstance("host1", 0); + OutstandingRequest req1 = tracker.newRequest(ni, 0); + Resource res0 = factory.newResource(48, 1); + + RoleStatus workerRole = lookupRole(WORKER); + // initial request + ContainerRequest yarnRequest = + req1.buildContainerRequest(res0, workerRole, 0); + assertEquals(req1.label, WORKERS_LABEL); + + assertNull(yarnRequest.getNodeLabelExpression()); + assertFalse(yarnRequest.getRelaxLocality()); + // escalation + ContainerRequest yarnRequest2 = req1.escalate(); + assertNull(yarnRequest2.getNodes()); + assertTrue(yarnRequest2.getRelaxLocality()); + assertEquals(yarnRequest2.getNodeLabelExpression(), WORKERS_LABEL); + } + + /** + * If the placement doesnt include a label, then the escalation request + * retains the node list, but sets relaxLocality==true. + * @throws Throwable + */ + @Test + public void testRequestUnlabelledPlacement() throws Throwable { + NodeInstance ni = new NodeInstance("host1", 0); + OutstandingRequest req1 = tracker.newRequest(ni, 0); + Resource res0 = factory.newResource(48, 1); + + // initial request + ContainerRequest yarnRequest = req1.buildContainerRequest(res0, + getRole0Status(), 0); + assertNotNull(yarnRequest.getNodes()); + assertTrue(SliderUtils.isUnset(yarnRequest.getNodeLabelExpression())); + assertFalse(yarnRequest.getRelaxLocality()); + ContainerRequest yarnRequest2 = req1.escalate(); + assertNotNull(yarnRequest2.getNodes()); + assertTrue(yarnRequest2.getRelaxLocality()); + } + + @Test(expected = IllegalArgumentException.class) + public void testAARequestNoNodes() throws Throwable { + tracker.newAARequest(getRole0Status().getKey(), new ArrayList<>(), ""); + } + + @Test + public void testAARequest() throws Throwable { + int role0 = getRole0Status().getKey(); + OutstandingRequest request = tracker.newAARequest(role0, Arrays + .asList(host1), ""); + assertEquals(host1.hostname, request.hostname); + assertFalse(request.isLocated()); + } + + @Test + public void testAARequestPair() throws Throwable { + int role0 = getRole0Status().getKey(); + OutstandingRequest request = tracker.newAARequest(role0, Arrays.asList( + host1, host2), ""); + assertEquals(host1.hostname, request.hostname); + assertFalse(request.isLocated()); + ContainerRequest yarnRequest = request.buildContainerRequest( + getRole0Status().copyResourceRequirements(new MockResource(0, 0)), + getRole0Status(), + 0); + assertFalse(yarnRequest.getRelaxLocality()); + assertFalse(request.mayEscalate()); + + assertEquals(2, yarnRequest.getNodes().size()); + } + + @Test + public void testBuildResourceRequirements() throws Throwable { + // Store original values + Application application = appState.getClusterStatus(); + Component role0 = application.getComponent(getRole0Status().getGroup()); + String origMem = role0.getResource().getMemory(); + Integer origVcores = role0.getResource().getCpus(); + + // Resource values to be used for this test + int testMem = 32768; + int testVcores = 2; + role0.resource(new org.apache.slider.api.resource.Resource().memory(Integer + .toString(testMem)).cpus(testVcores)); + + // Test normalization disabled + LOG.info("Test normalization: disabled"); + role0.getConfiguration().setProperty( + ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "false"); + MockResource requestedRes = new MockResource(testMem, testVcores); + MockResource expectedRes = new MockResource(testMem, testVcores); + LOG.info("Resource requested: {}", requestedRes); + Resource resFinal = appState.buildResourceRequirements(getRole0Status()); + LOG.info("Resource actual: {}", resFinal); + assertTrue(Resources.equals(expectedRes, resFinal)); + + // Test normalization enabled + LOG.info("Test normalization: enabled"); + role0.getConfiguration().setProperty( + ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "true"); + expectedRes = new MockResource(MockAppState.RM_MAX_RAM, testVcores); + LOG.info("Resource requested: {}", requestedRes); + resFinal = appState.buildResourceRequirements(getRole0Status()); + LOG.info("Resource actual: {}", resFinal); + assertTrue(Resources.equals(expectedRes, resFinal)); + + // revert resource configuration to original value + role0.resource(new org.apache.slider.api.resource.Resource().memory(origMem) + .cpus(origVcores)); + } + + public Resource newResource(RoleStatus r) { + return appState.buildResourceRequirements(r); + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java new file mode 100644 index 0000000..a936df5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.providers.PlacementPolicy; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.apache.slider.server.avro.LoadedRoleHistory; +import org.apache.slider.server.avro.RoleHistoryWriter; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Test fole history reading and writing. + */ +public class TestRoleHistoryRW extends BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryRW.class); + + private static long time = System.currentTimeMillis(); + public static final String HISTORY_V1_6_ROLE = + "org/apache/slider/server/avro/history-v01-6-role.json"; + public static final String HISTORY_V1_3_ROLE = + "org/apache/slider/server/avro/history-v01-3-role.json"; + public static final String HISTORY_V1B_1_ROLE = + "org/apache/slider/server/avro/history_v01b_1_role.json"; + + private RoleStatus role0Status; + private RoleStatus role1Status; + + static final ProviderRole PROVIDER_ROLE3 = new ProviderRole( + "role3", + 3, + PlacementPolicy.STRICT, + 3, + 3, + ResourceKeys.DEF_YARN_LABEL_EXPRESSION); + + @Override + public String getTestName() { + return "TestHistoryRW"; + } + + @Override + public void setup() throws Exception { + super.setup(); + role0Status = getRole0Status(); + role1Status = getRole1Status(); + } + + @Test + public void testWriteReadEmpty() throws Throwable { + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + roleHistory.onStart(fs, historyPath); + Path history = roleHistory.saveHistory(time++); + assertTrue(fs.getFileStatus(history).isFile()); + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + historyWriter.read(fs, history); + } + + @Test + public void testWriteReadData() throws Throwable { + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + assertFalse(roleHistory.onStart(fs, historyPath)); + String addr = "localhost"; + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr); + NodeEntry ne1 = instance.getOrCreate(0); + ne1.setLastUsed(0xf00d); + + Path history = roleHistory.saveHistory(time++); + assertTrue(fs.getFileStatus(history).isFile()); + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES); + + + LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history); + assertTrue(0 < loadedRoleHistory.size()); + rh2.rebuild(loadedRoleHistory); + NodeInstance ni2 = rh2.getExistingNodeInstance(addr); + assertNotNull(ni2); + NodeEntry ne2 = ni2.get(0); + assertNotNull(ne2); + assertEquals(ne2.getLastUsed(), ne1.getLastUsed()); + } + + @Test + public void testWriteReadActiveData() throws Throwable { + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + roleHistory.onStart(fs, historyPath); + String addr = "localhost"; + String addr2 = "rack1server5"; + NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr); + NodeEntry orig1 = localhost.getOrCreate(role0Status.getKey()); + orig1.setLastUsed(0x10); + NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2); + NodeEntry orig2 = rack1server5.getOrCreate(role1Status.getKey()); + orig2.setLive(3); + assertFalse(orig2.isAvailable()); + NodeEntry orig3 = localhost.getOrCreate(role1Status.getKey()); + orig3.setLastUsed(0x20); + orig3.setLive(1); + assertFalse(orig3.isAvailable()); + orig3.release(); + assertTrue(orig3.isAvailable()); + roleHistory.dump(); + + long savetime = 0x0001000; + Path history = roleHistory.saveHistory(savetime); + assertTrue(fs.getFileStatus(history).isFile()); + describe("Loaded"); + LOG.info("testWriteReadActiveData in {}", history); + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES); + LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history); + assertEquals(3, loadedRoleHistory.size()); + rh2.rebuild(loadedRoleHistory); + rh2.dump(); + + assertEquals(2, rh2.getClusterSize()); + NodeInstance ni2 = rh2.getExistingNodeInstance(addr); + assertNotNull(ni2); + NodeEntry loadedNE = ni2.get(role0Status.getKey()); + assertEquals(loadedNE.getLastUsed(), orig1.getLastUsed()); + NodeInstance ni2b = rh2.getExistingNodeInstance(addr2); + assertNotNull(ni2b); + NodeEntry loadedNE2 = ni2b.get(role1Status.getKey()); + assertNotNull(loadedNE2); + assertEquals(loadedNE2.getLastUsed(), savetime); + assertEquals(rh2.getThawedDataTime(), savetime); + + // now start it + rh2.buildRecentNodeLists(); + describe("starting"); + rh2.dump(); + List<NodeInstance> available0 = rh2.cloneRecentNodeList(role0Status + .getKey()); + assertEquals(1, available0.size()); + + NodeInstance entry = available0.get(0); + assertEquals(entry.hostname, "localhost"); + assertEquals(entry, localhost); + List<NodeInstance> available1 = rh2.cloneRecentNodeList(role1Status + .getKey()); + assertEquals(2, available1.size()); + //and verify that even if last used was set, the save time is picked up + assertEquals(entry.get(role1Status.getKey()).getLastUsed(), roleHistory + .getSaveTime()); + + } + + @Test + public void testWriteThaw() throws Throwable { + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + assertFalse(roleHistory.onStart(fs, historyPath)); + String addr = "localhost"; + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr); + NodeEntry ne1 = instance.getOrCreate(0); + ne1.setLastUsed(0xf00d); + + Path history = roleHistory.saveHistory(time++); + long savetime =roleHistory.getSaveTime(); + assertTrue(fs.getFileStatus(history).isFile()); + RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES); + assertTrue(rh2.onStart(fs, historyPath)); + NodeInstance ni2 = rh2.getExistingNodeInstance(addr); + assertNotNull(ni2); + NodeEntry ne2 = ni2.get(0); + assertNotNull(ne2); + assertEquals(ne2.getLastUsed(), ne1.getLastUsed()); + assertEquals(rh2.getThawedDataTime(), savetime); + } + + + @Test + public void testPurgeOlderEntries() throws Throwable { + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + time = 1; + Path file1 = touch(historyWriter, time++); + Path file2 = touch(historyWriter, time++); + Path file3 = touch(historyWriter, time++); + Path file4 = touch(historyWriter, time++); + Path file5 = touch(historyWriter, time++); + Path file6 = touch(historyWriter, time++); + + assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file1)); + assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file2)); + assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file2)); + assertEquals(3, historyWriter.purgeOlderHistoryEntries(fs, file5)); + assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file6)); + try { + // make an impossible assertion that will fail if the method + // actually completes + assertEquals(-1, historyWriter.purgeOlderHistoryEntries(fs, file1)); + } catch (FileNotFoundException ignored) { + // expected + } + + } + + public Path touch(RoleHistoryWriter historyWriter, long timeMs) + throws IOException { + Path path = historyWriter.createHistoryFilename(historyPath, timeMs); + FSDataOutputStream out = fs.create(path); + out.close(); + return path; + } + + @Test + public void testSkipEmptyFileOnRead() throws Throwable { + describe("verify that empty histories are skipped on read; old histories " + + "purged"); + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + roleHistory.onStart(fs, historyPath); + time = 0; + Path oldhistory = roleHistory.saveHistory(time++); + + String addr = "localhost"; + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr); + NodeEntry ne1 = instance.getOrCreate(0); + ne1.setLastUsed(0xf00d); + + Path goodhistory = roleHistory.saveHistory(time++); + + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + Path touched = touch(historyWriter, time++); + + RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES); + assertTrue(rh2.onStart(fs, historyPath)); + NodeInstance ni2 = rh2.getExistingNodeInstance(addr); + assertNotNull(ni2); + + //and assert the older file got purged + assertFalse(fs.exists(oldhistory)); + assertTrue(fs.exists(goodhistory)); + assertTrue(fs.exists(touched)); + } + + @Test + public void testSkipBrokenFileOnRead() throws Throwable { + describe("verify that empty histories are skipped on read; old histories " + + "purged"); + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + roleHistory.onStart(fs, historyPath); + time = 0; + Path oldhistory = roleHistory.saveHistory(time++); + + String addr = "localhost"; + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr); + NodeEntry ne1 = instance.getOrCreate(0); + ne1.setLastUsed(0xf00d); + + Path goodhistory = roleHistory.saveHistory(time++); + + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + Path badfile = historyWriter.createHistoryFilename(historyPath, time++); + FSDataOutputStream out = fs.create(badfile); + out.writeBytes("{broken:true}"); + out.close(); + + RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES); + describe("IGNORE STACK TRACE BELOW"); + + assertTrue(rh2.onStart(fs, historyPath)); + + describe("IGNORE STACK TRACE ABOVE"); + NodeInstance ni2 = rh2.getExistingNodeInstance(addr); + assertNotNull(ni2); + + //and assert the older file got purged + assertFalse(fs.exists(oldhistory)); + assertTrue(fs.exists(goodhistory)); + assertTrue(fs.exists(badfile)); + } + + /** + * Test that a v1 JSON file can be read. Here the number of roles + * matches the current state. + * @throws Throwable + */ + @Test + public void testReloadDataV13Role() throws Throwable { + String source = HISTORY_V1_3_ROLE; + RoleHistoryWriter writer = new RoleHistoryWriter(); + + LoadedRoleHistory loadedRoleHistory = writer.read(source); + assertEquals(4, loadedRoleHistory.size()); + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + assertEquals(0, roleHistory.rebuild(loadedRoleHistory)); + } + + /** + * Test that a v1 JSON file can be read. Here more roles than expected + * @throws Throwable + */ + @Test + public void testReloadDataV16Role() throws Throwable { + String source = HISTORY_V1_6_ROLE; + RoleHistoryWriter writer = new RoleHistoryWriter(); + + LoadedRoleHistory loadedRoleHistory = writer.read(source); + assertEquals(6, loadedRoleHistory.size()); + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + assertEquals(3, roleHistory.rebuild(loadedRoleHistory)); + } + + /** + * Test that a v1 JSON file can be read. Here the number of roles + * is less than the current state. + * @throws Throwable + */ + @Test + public void testReloadLessRoles() throws Throwable { + String source = HISTORY_V1_3_ROLE; + RoleHistoryWriter writer = new RoleHistoryWriter(); + + LoadedRoleHistory loadedRoleHistory = writer.read(source); + assertEquals(4, loadedRoleHistory.size()); + List<ProviderRole> expandedRoles = new ArrayList(MockFactory.ROLES); + expandedRoles.add(PROVIDER_ROLE3); + RoleHistory roleHistory = new MockRoleHistory(expandedRoles); + assertEquals(0, roleHistory.rebuild(loadedRoleHistory)); + } + + /** + * Test that a v1b JSON file can be read. Here more roles than expected + * @throws Throwable + */ + @Test + public void testReloadDataV1B1Role() throws Throwable { + String source = HISTORY_V1B_1_ROLE; + RoleHistoryWriter writer = new RoleHistoryWriter(); + + LoadedRoleHistory loadedRoleHistory = writer.read(source); + assertEquals(1, loadedRoleHistory.size()); + assertEquals(2, loadedRoleHistory.roleMap.size()); + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + assertEquals(0, roleHistory.rebuild(loadedRoleHistory)); + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java new file mode 100644 index 0000000..0bc2282 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.avro.NewerFilesFirst; +import org.apache.slider.server.avro.RoleHistoryWriter; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Test role history rw ordering. + */ +public class TestRoleHistoryRWOrdering extends BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryRWOrdering.class); + + private List<Path> paths = pathlist( + Arrays.asList( + "hdfs://localhost/history-0406c.json", + "hdfs://localhost/history-5fffa.json", + "hdfs://localhost/history-0001a.json", + "hdfs://localhost/history-0001f.json" + ) + ); + private Path h0406c = paths.get(0); + private Path h5fffa = paths.get(1); + private Path h0001a = paths.get(3); + + public TestRoleHistoryRWOrdering() throws URISyntaxException { + } + + List<Path> pathlist(List<String> pathnames) throws URISyntaxException { + List<Path> pathList = new ArrayList<>(); + for (String p : pathnames) { + pathList.add(new Path(new URI(p))); + } + return pathList; + } + + @Override + public String getTestName() { + return "TestHistoryRWOrdering"; + } + + /** + * This tests regexp pattern matching. It uses the current time so isn't + * repeatable -but it does test a wider range of values in the process + * @throws Throwable + */ + @Test + public void testPatternRoundTrip() throws Throwable { + describe("test pattern matching of names"); + long value=System.currentTimeMillis(); + String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN, + value); + String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN; + Pattern pattern = Pattern.compile(matchpattern); + Matcher matcher = pattern.matcher(name); + if (!matcher.find()) { + throw new Exception("No match for pattern $matchpattern in $name"); + } + } + + @Test + public void testWriteSequenceReadData() throws Throwable { + describe("test that if multiple entries are written, the newest is picked" + + " up"); + long time = System.currentTimeMillis(); + + RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + assertFalse(roleHistory.onStart(fs, historyPath)); + String addr = "localhost"; + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr); + NodeEntry ne1 = instance.getOrCreate(0); + ne1.setLastUsed(0xf00d); + + Path history1 = roleHistory.saveHistory(time++); + Path history2 = roleHistory.saveHistory(time++); + Path history3 = roleHistory.saveHistory(time); + + //inject a later file with a different name + sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello," + + " world"); + + + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + + List<Path> entries = historyWriter.findAllHistoryEntries( + fs, + historyPath, + false); + assertEquals(entries.size(), 3); + assertEquals(entries.get(0), history3); + assertEquals(entries.get(1), history2); + assertEquals(entries.get(2), history1); + } + + @Test + public void testPathStructure() throws Throwable { + assertEquals(h5fffa.getName(), "history-5fffa.json"); + } + + @Test + public void testPathnameComparator() throws Throwable { + + NewerFilesFirst newerName = new NewerFilesFirst(); + + LOG.info("{} name is {}", h5fffa, h5fffa.getName()); + LOG.info("{} name is {}", h0406c, h0406c.getName()); + assertEquals(newerName.compare(h5fffa, h5fffa), 0); + assertTrue(newerName.compare(h5fffa, h0406c) < 0); + assertTrue(newerName.compare(h5fffa, h0001a) < 0); + assertTrue(newerName.compare(h0001a, h5fffa) > 0); + + } + + @Test + public void testPathSort() throws Throwable { + List<Path> paths2 = new ArrayList<>(paths); + RoleHistoryWriter.sortHistoryPaths(paths2); + assertListEquals(paths2, + Arrays.asList( + paths.get(1), + paths.get(0), + paths.get(3), + paths.get(2) + )); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java new file mode 100644 index 0000000..7364201 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.providers.PlacementPolicy; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockContainer; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; +import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.OutstandingRequest; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +/** + * Test the RH availability list and request tracking: that hosts + * get removed and added. + */ +public class TestRoleHistoryRequestTracking extends BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryRequestTracking.class); + + private String roleName = "test"; + + private NodeInstance age1Active4; + private NodeInstance age2Active2; + private NodeInstance age2Active0; + private NodeInstance age3Active0; + private NodeInstance age4Active1; + + private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + // 1MB, 1 vcore + private Resource resource = Resource.newInstance(1, 1); + + private RoleStatus roleStatus; + + public TestRoleHistoryRequestTracking() throws BadConfigException { + } + + AMRMClient.ContainerRequest requestContainer(RoleStatus rs) { + return roleHistory.requestContainerForRole(rs).getIssuedRequest(); + } + + @Override + public String getTestName() { + return "TestRoleHistoryAvailableList"; + } + + @Override + public void setup() throws Exception { + super.setup(); + + age1Active4 = nodeInstance(1, 4, 0, 0); + age2Active2 = nodeInstance(2, 2, 0, 1); + age2Active0 = nodeInstance(2, 0, 0, 0); + age3Active0 = nodeInstance(3, 0, 0, 0); + age4Active1 = nodeInstance(4, 1, 0, 0); + + roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1, + age1Active4, age3Active0)); + roleHistory.buildRecentNodeLists(); + roleStatus = getRole0Status(); + roleStatus.setResourceRequirements(Resource.newInstance(1, 1)); + } + + @Test + public void testAvailableListBuiltForRoles() throws Throwable { + List<NodeInstance> available0 = roleHistory.cloneRecentNodeList( + roleStatus.getKey()); + assertListEquals(Arrays.asList(age3Active0, age2Active0), available0); + } + + @Test + public void testRequestedNodeOffList() throws Throwable { + NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus); + assertEquals(age3Active0, ni); + assertListEquals(Arrays.asList(age2Active0), + roleHistory.cloneRecentNodeList(roleStatus.getKey())); + roleHistory.requestInstanceOnNode(ni, + roleStatus, + resource + ); + } + + @Test + public void testRequestedNodeOffListWithFailures() throws Throwable { + assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty()); + + NodeEntry age3role0 = recordAsFailed(age3Active0, roleStatus.getKey(), 4); + assertTrue(age3Active0.isConsideredUnreliable(roleStatus.getKey(), + roleStatus.getNodeFailureThreshold())); + recordAsFailed(age2Active0, roleStatus.getKey(), 4); + assertTrue(age2Active0.isConsideredUnreliable(roleStatus.getKey(), + roleStatus.getNodeFailureThreshold())); + // expect to get a null node back + NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus); + assertNull(ni); + + // which is translated to a no-location request + AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni, + roleStatus, + resource).getIssuedRequest(); + + assertNull(req.getNodes()); + + LOG.info("resetting failure count"); + age3role0.resetFailedRecently(); + roleHistory.dump(); + assertEquals(0, age3role0.getFailedRecently()); + assertFalse(age3Active0.isConsideredUnreliable(roleStatus.getKey(), + roleStatus.getNodeFailureThreshold())); + assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty()); + // looking for a node should now find one + ni = roleHistory.findRecentNodeForNewInstance(roleStatus); + assertEquals(ni, age3Active0); + req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource) + .getIssuedRequest(); + assertEquals(1, req.getNodes().size()); + } + + /** + * Verify that strict placement policies generate requests for nodes + * irrespective of their failed status. + * @throws Throwable + */ + @Test + public void testStrictPlacementIgnoresFailures() throws Throwable { + + RoleStatus targetRole = getRole1Status(); + final ProviderRole providerRole1 = targetRole.getProviderRole(); + assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT); + int key1 = targetRole.getKey(); + int key0 = getRole0Status().getKey(); + + List<NodeInstance> nodes0 = Arrays.asList(age1Active4, age2Active0, + age2Active2, age3Active0, age4Active1); + recordAllFailed(key0, 4, nodes0); + recordAllFailed(key1, 4, nodes0); + + // trigger a list rebuild + roleHistory.buildRecentNodeLists(); + List<NodeInstance> recentRole0 = roleHistory.cloneRecentNodeList(key0); + assertTrue(recentRole0.indexOf(age3Active0) < recentRole0 + .indexOf(age2Active0)); + + // the non-strict role has no suitable nodes + assertNull(roleHistory.findRecentNodeForNewInstance(getRole0Status())); + + + NodeInstance ni = roleHistory.findRecentNodeForNewInstance(targetRole); + assertNotNull(ni); + + NodeInstance ni2 = roleHistory.findRecentNodeForNewInstance(targetRole); + assertNotNull(ni2); + assertNotEquals(ni, ni2); + } + + @Test + public void testFindAndRequestNode() throws Throwable { + AMRMClient.ContainerRequest req = requestContainer(roleStatus); + + assertEquals(age3Active0.hostname, req.getNodes().get(0)); + List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus + .getKey()); + assertListEquals(Arrays.asList(age2Active0), a2); + } + + @Test + public void testRequestedNodeIntoReqList() throws Throwable { + requestContainer(roleStatus); + List<OutstandingRequest> requests = roleHistory.listPlacedRequests(); + assertEquals(1, requests.size()); + assertEquals(age3Active0.hostname, requests.get(0).hostname); + } + + @Test + public void testCompletedRequestDropsNode() throws Throwable { + AMRMClient.ContainerRequest req = requestContainer(roleStatus); + List<OutstandingRequest> requests = roleHistory.listPlacedRequests(); + assertEquals(1, requests.size()); + String hostname = requests.get(0).hostname; + assertEquals(age3Active0.hostname, hostname); + assertEquals(hostname, req.getNodes().get(0)); + MockContainer container = factory.newContainer(req, hostname); + assertOnContainerAllocated(container, 2, 1); + assertNoOutstandingPlacedRequests(); + } + + public void assertOnContainerAllocated(Container c1, int p1, int p2) { + assertNotEquals(ContainerAllocationOutcome.Open, roleHistory + .onContainerAllocated(c1, p1, p2).outcome); + } + + public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) { + assertEquals(ContainerAllocationOutcome.Open, roleHistory + .onContainerAllocated(c1, p1, p2).outcome); + } + + void assertNoOutstandingPlacedRequests() { + assertTrue(roleHistory.listPlacedRequests().isEmpty()); + } + + public void assertOutstandingPlacedRequests(int i) { + assertEquals(i, roleHistory.listPlacedRequests().size()); + } + + @Test + public void testTwoRequests() throws Throwable { + AMRMClient.ContainerRequest req = requestContainer(roleStatus); + AMRMClient.ContainerRequest req2 = requestContainer(roleStatus); + List<OutstandingRequest> requests = roleHistory.listPlacedRequests(); + assertEquals(2, requests.size()); + MockContainer container = factory.newContainer(req, req.getNodes().get(0)); + assertOnContainerAllocated(container, 2, 1); + assertOutstandingPlacedRequests(1); + container = factory.newContainer(req2, req2.getNodes().get(0)); + assertOnContainerAllocated(container, 2, 2); + assertNoOutstandingPlacedRequests(); + } + + @Test + public void testThreeRequestsOneUnsatisified() throws Throwable { + AMRMClient.ContainerRequest req = requestContainer(roleStatus); + AMRMClient.ContainerRequest req2 = requestContainer(roleStatus); + AMRMClient.ContainerRequest req3 = requestContainer(roleStatus); + List<OutstandingRequest> requests = roleHistory.listPlacedRequests(); + assertEquals(2, requests.size()); + MockContainer container = factory.newContainer(req, req.getNodes().get(0)); + assertOnContainerAllocated(container, 2, 1); + assertOutstandingPlacedRequests(1); + + container = factory.newContainer(req3, "three"); + assertOnContainerAllocationOpen(container, 3, 2); + assertOutstandingPlacedRequests(1); + + // the final allocation will trigger a cleanup + container = factory.newContainer(req2, "four"); + // no node dropped + assertEquals(ContainerAllocationOutcome.Unallocated, + roleHistory.onContainerAllocated(container, 3, 3).outcome); + // yet the list is now empty + assertNoOutstandingPlacedRequests(); + roleHistory.listOpenRequests().isEmpty(); + + // and the remainder goes onto the available list + List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus + .getKey()); + assertListEquals(Arrays.asList(age2Active0), a2); + } + + @Test + public void testThreeRequests() throws Throwable { + AMRMClient.ContainerRequest req = requestContainer(roleStatus); + AMRMClient.ContainerRequest req2 = requestContainer(roleStatus); + AMRMClient.ContainerRequest req3 = requestContainer(roleStatus); + assertOutstandingPlacedRequests(2); + assertNull(req3.getNodes()); + MockContainer container = factory.newContainer(req, req.getNodes().get(0)); + assertOnContainerAllocated(container, 3, 1); + assertOutstandingPlacedRequests(1); + container = factory.newContainer(req2, req2.getNodes().get(0)); + assertOnContainerAllocated(container, 3, 2); + assertNoOutstandingPlacedRequests(); + container = factory.newContainer(req3, "three"); + assertOnContainerAllocationOpen(container, 3, 3); + assertNoOutstandingPlacedRequests(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java new file mode 100644 index 0000000..33e7930 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.server.appmaster.actions.ResetFailureWindow; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockAM; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler; +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +/** + * Test updating blacklist. + */ +public class TestRoleHistoryUpdateBlacklist extends BaseMockAppStateTest { + private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + private Collection<RoleStatus> roleStatuses; + private RoleStatus roleStatus; + private NodeInstance ni; + + public TestRoleHistoryUpdateBlacklist() throws BadConfigException { + } + + @Override + public String getTestName() { + return "TestUpdateBlacklist"; + } + + @Override + public void setup() throws Exception { + super.setup(); + ni = nodeInstance(1, 0, 0, 0); + roleHistory.insert(Arrays.asList(ni)); + roleHistory.buildRecentNodeLists(); + appState.setRoleHistory(roleHistory); + roleStatus = getRole0Status(); + roleStatuses = Arrays.asList(roleStatus); + } + + @Test + public void testUpdateBlacklist() { + assertFalse(ni.isBlacklisted()); + + // at threshold, blacklist is unmodified + recordAsFailed(ni, roleStatus.getKey(), MockFactory.NODE_FAILURE_THRESHOLD); + UpdateBlacklistOperation op = roleHistory.updateBlacklist(roleStatuses); + assertNull(op); + assertFalse(ni.isBlacklisted()); + + // threshold is reached, node goes on blacklist + recordAsFailed(ni, roleStatus.getKey(), 1); + op = roleHistory.updateBlacklist(roleStatuses); + assertNotNull(op); + assertTrue(ni.isBlacklisted()); + + // blacklist remains unmodified + op = roleHistory.updateBlacklist(roleStatuses); + assertNull(op); + assertTrue(ni.isBlacklisted()); + + // failure threshold reset, node goes off blacklist + ni.resetFailedRecently(); + op = roleHistory.updateBlacklist(roleStatuses); + assertNotNull(op); + assertFalse(ni.isBlacklisted()); + } + + @Test + public void testBlacklistOperations() + throws Exception { + recordAsFailed(ni, roleStatus.getKey(), MockFactory + .NODE_FAILURE_THRESHOLD + 1); + + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); + assertListLength(ops, 1); + AbstractRMOperation op = ops.get(0); + assertTrue(op instanceof UpdateBlacklistOperation); + assertTrue(ni.isBlacklisted()); + + MockRMOperationHandler handler = new MockRMOperationHandler(); + assertEquals(0, handler.getBlacklisted()); + handler.execute(ops); + assertEquals(1, handler.getBlacklisted()); + + ResetFailureWindow resetter = new ResetFailureWindow(handler); + resetter.execute(new MockAM(), null, appState); + assertEquals(0, handler.getBlacklisted()); + assertFalse(ni.isBlacklisted()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java new file mode 100644 index 0000000..419f2fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.common.tools.SliderUtils; + +/** + * Provides allocation services to a cluster -both random and placed. + * + * Important: container allocations need an app attempt ID put into the + * container ID + */ +public class Allocator { + + private final MockYarnCluster cluster; + /** + * Rolling index into the cluster used for the next "random" assignment. + */ + private int rollingIndex = 0; + + Allocator(MockYarnCluster cluster) { + this.cluster = cluster; + } + + /** + * Allocate a node using the list of nodes in the container as the + * hints. + * @param request request + * @return the allocated container -or null for none + */ + MockContainer allocate(AMRMClient.ContainerRequest request) { + MockYarnCluster.MockYarnClusterNode node = null; + MockYarnCluster.MockYarnClusterContainer allocated = null; + if (SliderUtils.isNotEmpty(request.getNodes())) { + for (String host : request.getNodes()) { + node = cluster.lookup(host); + allocated = node.allocate(); + if (allocated != null) { + break; + } + } + } + + if (allocated != null) { + return createContainerRecord(request, allocated, node); + } else { + if (request.getRelaxLocality() || request.getNodes().isEmpty()) { + // fallback to anywhere + return allocateRandom(request); + } else { + //no match and locality can't be requested + return null; + } + } + } + + /** + * Allocate a node without any positioning -use whatever policy this allocator + * chooses. + * @param request request + * @return the allocated container -or null for none + */ + MockContainer allocateRandom(AMRMClient.ContainerRequest request) { + int start = rollingIndex; + MockYarnCluster.MockYarnClusterNode node = cluster.nodeAt(rollingIndex); + MockYarnCluster.MockYarnClusterContainer allocated = node.allocate(); + // if there is no space, try again -but stop when all the nodes + // have failed + while (allocated == null && start != nextIndex()) { + node = cluster.nodeAt(rollingIndex); + allocated = node.allocate(); + } + + //here the allocation is set, so create the response + return createContainerRecord(request, allocated, node); + } + + /** + * Create a container record -if one was allocated. + * @param allocated allocation -may be null + * @param node node with the container + * @return a container record, or null if there was no allocation + */ + public MockContainer createContainerRecord( + AMRMClient.ContainerRequest request, + MockYarnCluster.MockYarnClusterContainer allocated, + MockYarnCluster.MockYarnClusterNode node) { + if (allocated == null) { + // no space + return null; + } + MockContainer container = new MockContainer(); + container.setId(new MockContainerId(allocated.getCid())); + container.setNodeId(node.getNodeId()); + container.setNodeHttpAddress(node.httpAddress()); + container.setPriority(request.getPriority()); + container.setResource(request.getCapability()); + return container; + } + + public int nextIndex() { + rollingIndex = (rollingIndex + 1) % cluster.getClusterSize(); + return rollingIndex; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java new file mode 100644 index 0000000..eca8401 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.api.resource.Application; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.exceptions.SliderInternalStateException; +import org.apache.slider.core.exceptions.TriggerClusterTeardownException; +import org.apache.slider.core.main.LauncherExitCodes; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.CancelSingleRequest; +import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; +import org.apache.slider.server.appmaster.state.ContainerAssignment; +import org.apache.slider.server.appmaster.state.ContainerOutcome; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.NodeMap; +import org.apache.slider.server.appmaster.state.ProviderAppState; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.utils.SliderTestBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map.Entry; + +/** + * Base for app state tests. + */ +public abstract class BaseMockAppStateTest extends SliderTestBase implements + MockRoles { + private static final Logger LOG = + LoggerFactory.getLogger(BaseMockAppStateTest.class); + protected static final List<ContainerId> EMPTY_ID_LIST = Collections + .emptyList(); + + protected final MockFactory factory = MockFactory.INSTANCE; + protected MockAppState appState; + protected MockYarnEngine engine; + protected FileSystem fs; + protected SliderFileSystem sliderFileSystem; + protected File historyWorkDir; + protected Path historyPath; + protected MockApplicationId applicationId; + protected MockApplicationAttemptId applicationAttemptId; + protected StateAccessForProviders stateAccess; + + /** + * Override point: called in setup() to create the YARN engine; can + * be changed for different sizes and options. + * @return + */ + public MockYarnEngine createYarnEngine() { + return new MockYarnEngine(8, 8); + } + + @Override + public void setup() throws Exception { + super.setup(); + YarnConfiguration conf = SliderUtils.createConfiguration(); + fs = FileSystem.get(new URI("file:///"), conf); + sliderFileSystem = new SliderFileSystem(fs, conf); + engine = createYarnEngine(); + initApp(); + } + + /** + * Initialize the application. + * This uses the binding information supplied by {@link #buildBindingInfo()}. + */ + protected void initApp() + throws IOException, BadConfigException, BadClusterStateException { + String historyDirName = getTestName(); + applicationId = new MockApplicationId(1, 0); + applicationAttemptId = new MockApplicationAttemptId(applicationId, 1); + + historyWorkDir = new File("target/history", historyDirName); + historyPath = new Path(historyWorkDir.toURI()); + fs.delete(historyPath, true); + appState = new MockAppState(buildBindingInfo()); + stateAccess = new ProviderAppState(getTestName(), appState); + } + + /** + * Build the binding info from the default constructor values, + * the roles from {@link #factory}, and an instance definition. + * from {@link #buildApplication()} ()} + * @return + */ + protected AppStateBindingInfo buildBindingInfo() { + AppStateBindingInfo binding = new AppStateBindingInfo(); + binding.application = buildApplication(); + //binding.roles = new ArrayList<>(factory.ROLES); + binding.fs = fs; + binding.historyPath = historyPath; + binding.nodeReports = engine.getNodeReports(); + return binding; + } + + /** + * Override point, define the instance definition. + * @return the instance definition + */ + public Application buildApplication() { + return factory.newApplication(0, 0, 0).name(getTestName()); + } + + /** + * Get the test name ... defaults to method name + * @return the method name + */ + public String getTestName() { + return methodName.getMethodName(); + } + + public RoleStatus getRole0Status() { + return lookupRole(ROLE0); + } + + public RoleStatus lookupRole(String role) { + return appState.lookupRoleStatus(role); + } + + public RoleStatus getRole1Status() { + return lookupRole(ROLE1); + } + + public RoleStatus getRole2Status() { + return lookupRole(ROLE2); + } + + /** + * Build a role instance from a container assignment. + * @param assigned + * @return the instance + */ + public RoleInstance roleInstance(ContainerAssignment assigned) { + Container target = assigned.container; + RoleInstance ri = new RoleInstance(target); + ri.roleId = assigned.role.getPriority(); + ri.role = assigned.role.getName(); + return ri; + } + + public NodeInstance nodeInstance(long age, int live0, int live1, int live2) { + NodeInstance ni = new NodeInstance(String.format("age%d-[%d,%d,%d]", age, + live0, live1, live2), MockFactory.ROLE_COUNT); + ni.getOrCreate(getRole0Status().getKey()).setLastUsed(age); + ni.getOrCreate(getRole0Status().getKey()).setLive(live0); + if (live1 > 0) { + ni.getOrCreate(getRole1Status().getKey()).setLive(live1); + } + if (live2 > 0) { + ni.getOrCreate(getRole2Status().getKey()).setLive(live2); + } + return ni; + } + + /** + * Create a container status event. + * @param c container + * @return a status + */ + ContainerStatus containerStatus(Container c) { + return containerStatus(c.getId()); + } + + /** + * Create a container status instance for the given ID, declaring + * that it was shut down by the application itself. + * @param cid container Id + * @return the instance + */ + public ContainerStatus containerStatus(ContainerId cid) { + ContainerStatus status = containerStatus(cid, + LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN); + return status; + } + + public ContainerStatus containerStatus(ContainerId cid, int exitCode) { + ContainerStatus status = ContainerStatus.newInstance( + cid, + ContainerState.COMPLETE, + "", + exitCode); + return status; + } + + /** + * Create nodes and bring them to the started state. + * @return a list of roles + */ + protected List<RoleInstance> createAndStartNodes() + throws TriggerClusterTeardownException, SliderInternalStateException { + return createStartAndStopNodes(new ArrayList<>()); + } + + /** + * Create, Start and stop nodes. + * @param completionResults List filled in with the status on all completed + * nodes + * @return the nodes + */ + public List<RoleInstance> createStartAndStopNodes( + List<AppState.NodeCompletionResult> completionResults) + throws TriggerClusterTeardownException, SliderInternalStateException { + List<ContainerId> released = new ArrayList<>(); + List<RoleInstance> instances = createAndSubmitNodes(released); + processSubmissionOperations(instances, completionResults, released); + return instances; + } + + /** + * Process the start/stop operations. + * @param instances + * @param completionResults + * @param released + */ + public void processSubmissionOperations( + List<RoleInstance> instances, + List<AppState.NodeCompletionResult> completionResults, + List<ContainerId> released) { + for (RoleInstance instance : instances) { + LOG.debug("Started {} on {}", instance.role, instance.id); + assertNotNull(appState.onNodeManagerContainerStarted(instance + .getContainerId())); + } + releaseContainers(completionResults, + released, + ContainerState.COMPLETE, + "released", + 0 + ); + } + + /** + * Release a list of containers, updating the completion results. + * @param completionResults + * @param containerIds + * @param containerState + * @param exitText + * @param containerExitCode + * @return + */ + public void releaseContainers( + List<AppState.NodeCompletionResult> completionResults, + List<ContainerId> containerIds, + ContainerState containerState, + String exitText, + int containerExitCode) { + for (ContainerId id : containerIds) { + ContainerStatus status = ContainerStatus.newInstance(id, + containerState, + exitText, + containerExitCode); + completionResults.add(appState.onCompletedContainer(status)); + } + } + + /** + * Create nodes and submit them. + * @return a list of roles + */ + public List<RoleInstance> createAndSubmitNodes() + throws TriggerClusterTeardownException, SliderInternalStateException { + return createAndSubmitNodes(new ArrayList<>()); + } + + /** + * Create nodes and submit them. + * @return a list of roles + */ + public List<RoleInstance> createAndSubmitNodes(List<ContainerId> containerIds) + throws TriggerClusterTeardownException, SliderInternalStateException { + return createAndSubmitNodes(containerIds, new ArrayList<>()); + } + + /** + * Create nodes and submit them. + * @return a list of roles allocated + */ + public List<RoleInstance> createAndSubmitNodes( + List<ContainerId> containerIds, + List<AbstractRMOperation> operationsOut) + throws TriggerClusterTeardownException, SliderInternalStateException { + List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes(); + return submitOperations(ops, containerIds, operationsOut); + } + + public List<RoleInstance> submitOperations( + List<AbstractRMOperation> operationsIn, + List<ContainerId> released) { + return submitOperations(operationsIn, released, new ArrayList<>()); + } + + /** + * Process the RM operations and send <code>onContainersAllocated</code> + * events to the app state. + * @param operationsIn list of incoming ops + * @param released released containers + * @return list of outbound operations + */ + public List<RoleInstance> submitOperations( + List<AbstractRMOperation> operationsIn, + List<ContainerId> released, + List<AbstractRMOperation> operationsOut) { + List<Container> allocatedContainers = engine.execute(operationsIn, + released); + List<ContainerAssignment> assignments = new ArrayList<>(); + appState.onContainersAllocated(allocatedContainers, assignments, + operationsOut); + + List<RoleInstance> roles = new ArrayList<>(); + for (ContainerAssignment assigned : assignments) { + Container container = assigned.container; + RoleInstance ri = roleInstance(assigned); + //tell the app it arrived + LOG.debug("Start submitted {} on ${}", ri.role, container.getId()); + appState.containerStartSubmitted(container, ri); + roles.add(ri); + } + return roles; + } + + /** + * Add the AM to the app state. + */ + protected void addAppMastertoAppState() { +// appState.buildAppMasterNode( +// new MockContainerId(applicationAttemptId, 999999L), +// "appmaster", +// 0, +// null); + } + + /** + * Extract the list of container IDs from the list of role instances. + * @param instances instance list + * @param role role to look up + * @return the list of CIDs + */ + public List<ContainerId> extractContainerIds( + List<RoleInstance> instances, + String role) { + List<ContainerId> ids = new ArrayList<>(); + for (RoleInstance ri : instances) { + if (ri.role.equals(role)) { + ids.add(ri.getContainerId()); + } + } + return ids; + } + + /** + * Record a node as failing. + * @param node + * @param id + * @param count + * @return the entry + */ + public NodeEntry recordAsFailed(NodeInstance node, int id, int count) { + NodeEntry entry = node.getOrCreate(id); + for (int i = 1; i <= count; i++) { + entry.containerCompleted( + false, + ContainerOutcome.Failed); + } + return entry; + } + + protected void recordAllFailed(int id, int count, List<NodeInstance> nodes) { + for (NodeInstance node : nodes) { + recordAsFailed(node, id, count); + } + } + + /** + * Get the container request of an indexed entry. Includes some assertions + * for better diagnostics + * @param ops operation list + * @param index index in the list + * @return the request. + */ + public AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops, + int index) { + assertTrue(index < ops.size()); + AbstractRMOperation op = ops.get(index); + assertTrue(op instanceof ContainerRequestOperation); + return ((ContainerRequestOperation) op).getRequest(); + } + + /** + * Get the cancel request of an indexed entry. Includes some assertions for + * better diagnostics + * @param ops operation list + * @param index index in the list + * @return the request. + */ + public AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops, + int index) { + assertTrue(index < ops.size()); + AbstractRMOperation op = ops.get(index); + assertTrue(op instanceof CancelSingleRequest); + return ((CancelSingleRequest) op).getRequest(); + } + + /** + * Get the single request of a list of operations; includes the check for + * the size. + * @param ops operations list of size 1 + * @return the request within the first ContainerRequestOperation + */ + public AMRMClient.ContainerRequest getSingleRequest( + List<AbstractRMOperation> ops) { + assertEquals(1, ops.size()); + return getRequest(ops, 0); + } + + /** + * Get the single request of a list of operations; includes the check for + * the size. + * @param ops operations list of size 1 + * @return the request within the first operation + */ + public AMRMClient.ContainerRequest getSingleCancel( + List<AbstractRMOperation> ops) { + assertEquals(1, ops.size()); + return getCancel(ops, 0); + } + + /** + * Get the single release of a list of operations; includes the check for + * the size. + * @param ops operations list of size 1 + * @return the request within the first operation + */ + public ContainerReleaseOperation getSingleRelease( + List<AbstractRMOperation> ops) { + assertEquals(1, ops.size()); + AbstractRMOperation op = ops.get(0); + assertTrue(op instanceof ContainerReleaseOperation); + return (ContainerReleaseOperation) op; + } + + /** + * Get the node information as a large JSON String. + * @return + */ + protected String nodeInformationSnapshotAsString() + throws UnsupportedEncodingException, JsonProcessingException { + return prettyPrintAsJson(stateAccess.getNodeInformationSnapshot()); + } + + /** + * Scan through all containers and assert that the assignment is AA. + * @param index role index + */ + protected void assertAllContainersAA(int index) { + for (Entry<String, NodeInstance> nodeMapEntry : cloneNodemap().entrySet()) { + String name = nodeMapEntry.getKey(); + NodeInstance ni = nodeMapEntry.getValue(); + NodeEntry nodeEntry = ni.get(index); + assertTrue("too many instances on node " + name, nodeEntry == null || + nodeEntry.isAntiAffinityConstraintHeld()); + } + } + + /** + * Get a snapshot of the nodemap of the application state. + * @return a cloned nodemap + */ + protected NodeMap cloneNodemap() { + return appState.getRoleHistory().cloneNodemap(); + } + + /** + * Issue a nodes updated event. + * @param report report to notify + * @return response of AM + */ + protected AppState.NodeUpdatedOutcome updateNodes(NodeReport report) { + return appState.onNodesUpdated(Collections.singletonList(report)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java new file mode 100644 index 0000000..66ae0f9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.slider.server.appmaster.SliderAppMaster; + +/** + * Mock AM. + */ +public class MockAM extends SliderAppMaster { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java new file mode 100644 index 0000000..2fcf054 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.providers.ProviderRole; +import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; +import org.apache.slider.server.appmaster.state.AbstractClusterServices; +import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; + +import java.io.IOException; +import java.util.Map; + +/** + * Extended app state that makes more things public. + */ +public class MockAppState extends AppState { + public static final int RM_MAX_RAM = 4096; + public static final int RM_MAX_CORES = 64; + + private long time = -1; + + public MockAppState(AbstractClusterServices recordFactory) { + super(recordFactory, new MetricsAndMonitoring()); + setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES); + } + + /** + * Instance with a mock record factory. + */ + public MockAppState() { + this(new MockClusterServices()); + } + + public MockAppState(AppStateBindingInfo bindingInfo) + throws BadClusterStateException, IOException, BadConfigException { + this(); + buildInstance(bindingInfo); + } + + public Map<String, ProviderRole> getRoleMap() { + return super.getRoleMap(); + } + + /** + * Current time. if the <code>time</code> field + * is set, that value is returned + * @return the current time. + */ + protected long now() { + if (time > 0) { + return time; + } + return System.currentTimeMillis(); + } + + public void setTime(long newTime) { + this.time = newTime; + } + + public void incTime(long inc) { + this.time = this.time + inc; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java new file mode 100644 index 0000000..b509625 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +class MockApplicationAttemptId extends ApplicationAttemptId { + + private ApplicationId applicationId; + private int attemptId; + + public MockApplicationAttemptId() { + } + + public MockApplicationAttemptId(ApplicationId applicationId, int attemptId) { + this.applicationId = applicationId; + this.attemptId = attemptId; + } + + @Override + public ApplicationId getApplicationId() { + return applicationId; + } + + @Override + public void setApplicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + } + + @Override + public int getAttemptId() { + return attemptId; + } + + @Override + public void setAttemptId(int attemptId) { + this.attemptId = attemptId; + } + + @Override + protected void build() { + + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
