SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8cecce18 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8cecce18 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8cecce18 Branch: refs/heads/feature/SLIDER-151_REST_API Commit: 8cecce1821733c33de3cf9723f9a96aee411305d Parents: 5ce953c Author: Steve Loughran <ste...@apache.org> Authored: Wed Aug 6 17:11:39 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Aug 6 17:11:39 2014 +0100 ---------------------------------------------------------------------- .../server/appmaster/state/NodeInstance.java | 12 +- .../server/appmaster/state/RoleHistory.java | 14 + .../history/TestFindNodesForNewInstances.groovy | 128 --------- .../history/TestFindNodesForRelease.groovy | 146 ----------- .../model/history/TestHistoryRW.groovy | 258 ------------------- .../model/history/TestHistoryRWOrdering.groovy | 145 ----------- .../model/history/TestNIComparators.groovy | 77 ------ .../TestOutstandingRequestTracker.groovy | 73 ------ .../TestRoleHistoryContainerEvents.groovy | 17 +- ...stRoleHistoryFindNodesForNewInstances.groovy | 128 +++++++++ .../TestRoleHistoryFindNodesForRelease.groovy | 146 +++++++++++ .../history/TestRoleHistoryNIComparators.groovy | 77 ++++++ ...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++++++ .../model/history/TestRoleHistoryRW.groovy | 258 +++++++++++++++++++ .../history/TestRoleHistoryRWOrdering.groovy | 145 +++++++++++ 15 files changed, 861 insertions(+), 836 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java index 06375fb..fad60f2 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java @@ -75,7 +75,7 @@ public class NodeInstance { } /** - * Cout the number of active role instances on this node + * Count the number of active role instances on this node * @param role role index * @return 0 if there are none, otherwise the #of nodes that are running and * not being released already. @@ -84,6 +84,16 @@ public class NodeInstance { NodeEntry nodeEntry = get(role); return (nodeEntry != null ) ? nodeEntry.getActive() : 0; } + + /** + * Count the number of live role instances on this node + * @param role role index + * @return 0 if there are none, otherwise the #of nodes that are running + */ + public int getLiveRoleInstances(int role) { + NodeEntry nodeEntry = get(role); + return (nodeEntry != null ) ? nodeEntry.getLive() : 0; + } /** * Get the entry for a role -and remove it if present http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index de7e9d8..dbb104c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -529,6 +529,20 @@ public class RoleHistory { return nodemap.findNodesForRelease(role, count); } + + /** + * Get the list of active nodes ... walks the node map so + * is O(nodes) + * @param role role index + * @return a possibly empty list of nodes with an instance of that node + */ + public synchronized List<NodeInstance> listActiveNodes(int role) { + return nodemap.listActiveNodes(role); + } + + + + /** * Get the node entry of a container * @param container container to look up http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy deleted file mode 100644 index dab03f5..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.slider.providers.ProviderRole -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.state.NodeInstance -import org.apache.slider.server.appmaster.state.RoleHistory -import org.apache.slider.server.appmaster.state.RoleStatus -import org.junit.Before -import org.junit.Test - -/** - * Testing finding nodes for new instances. - * These tests validate the (currently) suboptimal - * behavior of not listing any known nodes when there - * are none in the available list -even if there are nodes - * known to be not running live instances in the cluster. - */ -@Slf4j -@CompileStatic -class TestFindNodesForNewInstances extends BaseMockAppStateTest { - - @Override - String getTestName() { - return "TestFindNodesForNewInstances" - } - - NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0) - NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1) - NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0) - NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0) - NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0) - NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) - - List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0] - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - - String roleName = "test" - RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0)) - RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2)) - - @Before - public void setupNodeMap() { - roleHistory.insert(nodes) - roleHistory.buildAvailableNodeLists(); - } - - - public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) { - List < NodeInstance > found = []; - for (int i = 0; i < count; i++) { - NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus) - if (f) { - found << f - }; - } - return found - } - - @Test - public void testFind1NodeR0() throws Throwable { - NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) - log.info("found: $found") - assert [age3Active0].contains(found) - } - - @Test - public void testFind2NodeR0() throws Throwable { - NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) - log.info("found: $found") - assert [age2Active0, age3Active0].contains(found) - NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat) - log.info("found: $found2") - assert [age2Active0, age3Active0].contains(found2) - assert found != found2; - } - @Test - public void testFind3NodeR0ReturnsNull() throws Throwable { - assert 2== findNodes(2).size() - NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) - assert found == null; - } - - @Test - public void testFindNodesOneEntry() throws Throwable { - List<NodeInstance> nodes = findNodes(4, roleStat2) - assert 0 == nodes.size() - } - - @Test - public void testFindNodesIndependent() throws Throwable { - assert 2 == findNodes(2).size() - roleHistory.dump() - assert 0 == findNodes(3, roleStat2).size() - } - - @Test - public void testFindNodesFallsBackWhenUsed() throws Throwable { - // mark age2 and active 0 as busy, expect a null back - age2Active0.get(0).onStartCompleted() - assert age2Active0.getActiveRoleInstances(0) != 0 - age3Active0.get(0).onStartCompleted() - assert age3Active0.getActiveRoleInstances(0) != 0 - NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) - log.info(found ?.toFullString()) - assert found == null - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy deleted file mode 100644 index 92915dd..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.state.NodeInstance -import org.apache.slider.server.appmaster.state.NodeMap -import org.junit.Before -import org.junit.Test - -@Slf4j -@CompileStatic -class TestFindNodesForRelease extends BaseMockAppStateTest { - - - @Override - String getTestName() { - return "TestFindNodesForRelease" - } - NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0) - NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0) - NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0) - NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0) - NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) - - List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0] - NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT); - - - @Before - public void setupNodeMap() { - nodeMap.insert(nodes) - } - - private void assertReleased( - int count, - List<NodeInstance> expected, - int role = 0) { - List<NodeInstance> released = nodeMap.findNodesForRelease(role, count) - assertListEquals(released, expected) - } - private void assertReleased( - List<NodeInstance> expected, - int role = 0) { - List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size()) - assertListEquals(released, expected) - } - - @Test - public void testListActiveNodes() throws Throwable { - assertListEquals(nodeMap.listActiveNodes(0), - [age1Active4,age2Active2, age4Active1]) - } - - @Test - public void testReleaseMinus1() throws Throwable { - try { - nodeMap.findNodesForRelease(0, -1) - fail("Expected an exception") - } catch (IllegalArgumentException e) { - } - } - @Test - public void testReleaseO() throws Throwable { - assertReleased(0, []) - } - - @Test - public void testRelease1() throws Throwable { - assertReleased(1, [age1Active4]) - } - - @Test - public void testRelease2() throws Throwable { - assertReleased(2, [age1Active4, age1Active4]) - } - - @Test - public void testRelease3() throws Throwable { - assertReleased(3, [age1Active4, age1Active4, age1Active4 ]) - } - - @Test - public void testRelease4() throws Throwable { - assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2]) - } - - @Test - public void testRelease5() throws Throwable { - assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1]) - } - - @Test - public void testRelease6() throws Throwable { - assertReleased( - [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4]) - } - - @Test - public void testRelease7() throws Throwable { - assertReleased( - [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, - age1Active4, age2Active2]) - } - - @Test - public void testRelease8() throws Throwable { - assertReleased(8, - [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, - age1Active4, age2Active2]) - } - - @Test - public void testPurgeInactiveTime3() throws Throwable { - assert nodeMap.purgeUnusedEntries(3) == 0; - } - - @Test - public void testPurgeInactiveTime4() throws Throwable { - assert nodeMap.purgeUnusedEntries(4) == 1; - } - @Test - public void testPurgeInactiveTime5() throws Throwable { - assert nodeMap.purgeUnusedEntries(5) == 1; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy deleted file mode 100644 index b646661..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import org.apache.hadoop.fs.FSDataOutputStream -import org.apache.hadoop.fs.Path -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.state.NodeEntry -import org.apache.slider.server.appmaster.state.NodeInstance -import org.apache.slider.server.appmaster.state.RoleHistory -import org.apache.slider.server.avro.RoleHistoryWriter -import org.junit.Test - -@Slf4j -@CompileStatic -class TestHistoryRW extends BaseMockAppStateTest { - - static long time = System.currentTimeMillis(); - - @Override - String getTestName() { - return "TestHistoryRW" - } - - @Test - public void testWriteReadEmpty() throws Throwable { - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - roleHistory.onStart(fs, historyPath) - Path history = roleHistory.saveHistory(time++) - assert fs.isFile(history) - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - historyWriter.read(fs, history, roleHistory) - } - - @Test - public void testWriteReadData() throws Throwable { - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - assert !roleHistory.onStart(fs, historyPath) - String addr = "localhost" - NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) - NodeEntry ne1 = instance.getOrCreate(0) - ne1.lastUsed = 0xf00d - - Path history = roleHistory.saveHistory(time++) - assert fs.isFile(history) - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) - - assert 0 < historyWriter.read(fs, history, rh2) - NodeInstance ni2 = rh2.getExistingNodeInstance(addr) - assert ni2 != null - NodeEntry ne2 = ni2.get(0) - assert ne2 !=null - assert ne2.lastUsed == ne1.lastUsed - } - - @Test - public void testWriteReadActiveData() throws Throwable { - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - roleHistory.onStart(fs, historyPath) - String addr = "localhost" - String addr2 = "rack1server5" - NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr) - NodeEntry orig1 = localhost.getOrCreate(0) - orig1.lastUsed = 0x10 - NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2) - NodeEntry orig2 = rack1server5.getOrCreate(1) - orig2.live = 3 - assert !orig2.available - NodeEntry orig3 = localhost.getOrCreate(1) - orig3.lastUsed = 0x20 - orig3.live = 1 - assert !orig3.available - orig3.release() - assert orig3.available - roleHistory.dump() - - long savetime = 0x0001000 - Path history = roleHistory.saveHistory(savetime) - assert fs.isFile(history) - describe("Loaded") - log.info("testWriteReadActiveData in $history") - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) - - assert 3 == historyWriter.read(fs, history, rh2) - rh2.dump() - - assert rh2.clusterSize == 2; - NodeInstance ni2 = rh2.getExistingNodeInstance(addr) - assert ni2 != null - NodeEntry loadedNE = ni2.get(0) - assert loadedNE.lastUsed == orig1.lastUsed - NodeInstance ni2b = rh2.getExistingNodeInstance(addr2) - assert ni2b != null - NodeEntry loadedNE2 = ni2b.get(1) - assert loadedNE2 != null - assert loadedNE2.lastUsed == savetime - assert rh2.thawedDataTime == savetime - - // now thaw it - rh2.buildAvailableNodeLists(); - describe("thawing") - rh2.dump(); - List<NodeInstance> available0 = rh2.cloneAvailableList(0) - assert available0.size() == 1 - - NodeInstance entry = available0.get(0) - assert entry.hostname == "localhost" - assert entry == localhost - List<NodeInstance> available1 = rh2.cloneAvailableList(1) - assert available1.size() == 2 - //and verify that even if last used was set, the save time is picked up - assert entry.get(1).lastUsed == roleHistory.saveTime - - } - - @Test - public void testWriteThaw() throws Throwable { - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - assert !roleHistory.onStart(fs, historyPath) - String addr = "localhost" - NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) - NodeEntry ne1 = instance.getOrCreate(0) - ne1.lastUsed = 0xf00d - - Path history = roleHistory.saveHistory(time++) - long savetime =roleHistory.saveTime; - assert fs.isFile(history) - RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) - assert rh2.onStart(fs, historyPath) - NodeInstance ni2 = rh2.getExistingNodeInstance(addr) - assert ni2 != null - NodeEntry ne2 = ni2.get(0) - assert ne2 != null - assert ne2.lastUsed == ne1.lastUsed - assert rh2.thawedDataTime == savetime - } - - - @Test - public void testPurgeOlderEntries() throws Throwable { - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - time = 1; - Path file1 = touch(historyWriter, time++) - Path file2 = touch(historyWriter, time++) - Path file3 = touch(historyWriter, time++) - Path file4 = touch(historyWriter, time++) - Path file5 = touch(historyWriter, time++) - Path file6 = touch(historyWriter, time++) - - assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0 - assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1 - assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0 - assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3 - assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1 - try { - // make an impossible assertion that will fail if the method - // actually completes - assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1) - } catch (FileNotFoundException ignored) { - // expected - } - - } - - public Path touch(RoleHistoryWriter historyWriter, long time){ - Path path = historyWriter.createHistoryFilename(historyPath, time); - FSDataOutputStream out = fs.create(path); - out.close() - return path - } - - @Test - public void testSkipEmptyFileOnRead() throws Throwable { - describe "verify that empty histories are skipped on read; old histories purged" - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - roleHistory.onStart(fs, historyPath) - time = 0 - Path oldhistory = roleHistory.saveHistory(time++) - - String addr = "localhost" - NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) - NodeEntry ne1 = instance.getOrCreate(0) - ne1.lastUsed = 0xf00d - - Path goodhistory = roleHistory.saveHistory(time++) - - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - Path touched = touch(historyWriter, time++) - - RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) - assert rh2.onStart(fs, historyPath) - NodeInstance ni2 = rh2.getExistingNodeInstance(addr) - assert ni2 != null - - //and assert the older file got purged - assert !fs.exists(oldhistory) - assert fs.exists(goodhistory) - assert fs.exists(touched ) - } - - @Test - public void testSkipBrokenFileOnRead() throws Throwable { - describe "verify that empty histories are skipped on read; old histories purged" - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - roleHistory.onStart(fs, historyPath) - time = 0 - Path oldhistory = roleHistory.saveHistory(time++) - - String addr = "localhost" - NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) - NodeEntry ne1 = instance.getOrCreate(0) - ne1.lastUsed = 0xf00d - - Path goodhistory = roleHistory.saveHistory(time++) - - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - Path badfile = historyWriter.createHistoryFilename(historyPath, time++) - FSDataOutputStream out = fs.create(badfile) - out.writeBytes("{broken:true}") - out.close() - - RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) - describe("IGNORE STACK TRACE BELOW") - - assert rh2.onStart(fs, historyPath) - - describe( "IGNORE STACK TRACE ABOVE") - NodeInstance ni2 = rh2.getExistingNodeInstance(addr) - assert ni2 != null - - //and assert the older file got purged - assert !fs.exists(oldhistory) - assert fs.exists(goodhistory) - assert fs.exists(badfile ) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy deleted file mode 100644 index 6ec046c..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history - -import groovy.util.logging.Slf4j -import org.apache.hadoop.fs.Path -import org.apache.slider.common.SliderKeys -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.state.NodeEntry -import org.apache.slider.server.appmaster.state.NodeInstance -import org.apache.slider.server.appmaster.state.RoleHistory -import org.apache.slider.server.avro.RoleHistoryWriter -import org.junit.Test - -import java.util.regex.Matcher -import java.util.regex.Pattern - -@Slf4j -class TestHistoryRWOrdering extends BaseMockAppStateTest { - - def paths = pathlist( - [ - "hdfs://localhost/history-0406c.json", - "hdfs://localhost/history-5fffa.json", - "hdfs://localhost/history-0001a.json", - "hdfs://localhost/history-0001f.json", - ] - ) - Path h_0406c = paths[0] - Path h_5fffa = paths[1] - Path h_0001a = paths[3] - - - List<Path> pathlist(List<String> pathnames) { - def result = [] - pathnames.each { result << new Path(new URI(it as String)) } - result - } - - @Override - String getTestName() { - return "TestHistoryRWOrdering" - } - - - /** - * This tests regexp pattern matching. It uses the current time so isn't - * repeatable -but it does test a wider range of values in the process - * @throws Throwable - */ - @Test - public void testPatternRoundTrip() throws Throwable { - describe "test pattern matching of names" - long value=System.currentTimeMillis() - String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value) - String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN - Pattern pattern = Pattern.compile(matchpattern) - Matcher matcher = pattern.matcher(name); - if (!matcher.find()) { - throw new Exception("No match for pattern $matchpattern in $name") - } - } - - - @Test - public void testWriteSequenceReadData() throws Throwable { - describe "test that if multiple entries are written, the newest is picked up" - long time = System.currentTimeMillis(); - - RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) - assert !roleHistory.onStart(fs, historyPath) - String addr = "localhost" - NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) - NodeEntry ne1 = instance.getOrCreate(0) - ne1.lastUsed = 0xf00d - - Path history1 = roleHistory.saveHistory(time++) - Path history2 = roleHistory.saveHistory(time++) - Path history3 = roleHistory.saveHistory(time++) - - //inject a later file with a different name - sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world") - - - RoleHistoryWriter historyWriter = new RoleHistoryWriter(); - - List<Path> entries = historyWriter.findAllHistoryEntries( - fs, - historyPath, - false) - assert entries.size() == 3 - assert entries[0] == history3 - assert entries[1] == history2 - assert entries[2] == history1 - } - - @Test - public void testPathStructure() throws Throwable { - assert h_5fffa.getName() == "history-5fffa.json" - } - - @Test - public void testPathnameComparator() throws Throwable { - - def newerName = new RoleHistoryWriter.NewerFilesFirst() - - log.info("$h_5fffa name is ${h_5fffa.getName()}") - log.info("$h_0406c name is ${h_0406c.getName()}") - assert newerName.compare(h_5fffa, h_5fffa) == 0 - assert newerName.compare(h_5fffa, h_0406c) < 0 - assert newerName.compare(h_5fffa, h_0001a) < 0 - assert newerName.compare(h_0001a, h_5fffa) > 0 - - } - - @Test - public void testPathSort() throws Throwable { - def paths2 = new ArrayList(paths) - RoleHistoryWriter.sortHistoryPaths(paths2) - assertListEquals(paths2, - [ - paths[1], - paths[0], - paths[3], - paths[2] - ]) - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy deleted file mode 100644 index 77119d5..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history - -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockFactory -import org.apache.slider.server.appmaster.state.NodeInstance -import org.junit.Test - -/** - * Unit test to verify the comparators sort as expected - */ -class TestNIComparators extends BaseMockAppStateTest { - - NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0) - NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0) - NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0) - NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0) - NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) - - List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0] - - @Override - String getTestName() { - return "TestNIComparators" - } - - @Test - public void testNewerThan() throws Throwable { - - Collections.sort(nodes, new NodeInstance.newerThan(0)) - assertListEquals(nodes, - [age4Active1, age3Active0, age2Active2, age1Active4]) - } - - @Test - public void testNewerThanNoRole() throws Throwable { - - nodes << empty - Collections.sort(nodes, new NodeInstance.newerThan(0)) - assertListEquals(nodes, - [age4Active1, age3Active0, age2Active2, age1Active4, empty]) - } - - @Test - public void testMoreActiveThan() throws Throwable { - - Collections.sort(nodes, new NodeInstance.moreActiveThan(0)) - assertListEquals(nodes, - [age1Active4, age2Active2, age4Active1, age3Active0],) - } - - @Test - public void testMoreActiveThanEmpty() throws Throwable { - nodes << empty - Collections.sort(nodes, new NodeInstance.moreActiveThan(0)) - assertListEquals(nodes, - [age1Active4, age2Active2, age4Active1, age3Active0, empty]) - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy deleted file mode 100644 index 8d1f4b0..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.history - -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.state.NodeInstance -import org.apache.slider.server.appmaster.state.OutstandingRequest -import org.apache.slider.server.appmaster.state.OutstandingRequestTracker -import org.junit.Test - -class TestOutstandingRequestTracker extends BaseMockAppStateTest { - - NodeInstance host1 = new NodeInstance("host1", 3) - NodeInstance host2 = new NodeInstance("host2", 3) - - OutstandingRequestTracker tracker = new OutstandingRequestTracker() - - @Override - String getTestName() { - return "TestOutstandingRequestTracker" - } - - @Test - public void testAddRetrieveEntry() throws Throwable { - OutstandingRequest request = tracker.addRequest(host1, 0) - assert tracker.lookup(0, "host1").equals(request) - assert tracker.remove(request).equals(request) - assert !tracker.lookup(0, "host1") - } - - @Test - public void testAddCompleteEntry() throws Throwable { - tracker.addRequest(host1, 0) - tracker.addRequest(host2, 0) - tracker.addRequest(host1, 1) - assert tracker.onContainerAllocated(1, "host1") - assert !tracker.lookup(1, "host1") - assert tracker.lookup(0, "host1") - } - - @Test - public void testCancelEntries() throws Throwable { - OutstandingRequest r1 = tracker.addRequest(host1, 0) - OutstandingRequest r2 = tracker.addRequest(host2, 0) - OutstandingRequest r3 = tracker.addRequest(host1, 1) - List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0) - assert canceled.size() == 2 - assert canceled.contains(host1) - assert canceled.contains(host2) - assert tracker.lookup(1, "host1") - assert !tracker.lookup(0, "host1") - canceled = tracker.cancelOutstandingRequests(0) - assert canceled.size() == 0 - assert tracker.cancelOutstandingRequests(1).size() == 1 - } - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy index 1a71aed..340e72d 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy @@ -114,12 +114,12 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { RoleStatus roleStatus = new RoleStatus(provRole) //verify it is empty - assert roleHistory.findNodesForRelease(role, 1).isEmpty() + assert roleHistory.listActiveNodes(role).empty AMRMClient.ContainerRequest request = roleHistory.requestNode(roleStatus, resource); - List<String> nodes = request.getNodes() + List<String> nodes = request.nodes assert nodes == null //pick an idle host @@ -128,7 +128,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { //build a container MockContainer container = factory.newContainer() container.nodeId = new MockNodeId(hostname, 0) - container.priority = request.getPriority() + container.priority = request.priority roleHistory.onContainerAssigned(container); NodeMap nodemap = roleHistory.cloneNodemap(); @@ -147,10 +147,11 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { assert roleEntry.live == 1 // now pick that instance to destroy + List<NodeInstance> activeNodes = roleHistory.listActiveNodes(role) + - List<NodeInstance> forRelease = roleHistory.findNodesForRelease(role, 1) - assert forRelease.size() == 1 - NodeInstance target = forRelease[0] + assert activeNodes.size() == 1 + NodeInstance target = activeNodes[0] assert target == allocated roleHistory.onContainerReleaseSubmitted(container); assert roleEntry.releasing == 1 @@ -164,13 +165,13 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { assert roleEntry.active == 0 // verify it is empty - assert roleHistory.findNodesForRelease(role, 1).isEmpty() + assert roleHistory.listActiveNodes(role).empty // ask for a container and expect to get the recently released one AMRMClient.ContainerRequest request2 = roleHistory.requestNode(roleStatus, resource); - List<String> nodes2 = request2.getNodes() + List<String> nodes2 = request2.nodes assert nodes2 != null String hostname2 = nodes2[0] http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy new file mode 100644 index 0000000..79cd348 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.providers.ProviderRole +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.state.NodeInstance +import org.apache.slider.server.appmaster.state.RoleHistory +import org.apache.slider.server.appmaster.state.RoleStatus +import org.junit.Before +import org.junit.Test + +/** + * Testing finding nodes for new instances. + * These tests validate the (currently) suboptimal + * behavior of not listing any known nodes when there + * are none in the available list -even if there are nodes + * known to be not running live instances in the cluster. + */ +@Slf4j +@CompileStatic +class TestRoleHistoryFindNodesForNewInstances extends BaseMockAppStateTest { + + @Override + String getTestName() { + return "TestFindNodesForNewInstances" + } + + NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0) + NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1) + NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0) + NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0) + NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0) + NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) + + List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0] + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + + String roleName = "test" + RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0)) + RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2)) + + @Before + public void setupNodeMap() { + roleHistory.insert(nodes) + roleHistory.buildAvailableNodeLists(); + } + + + public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) { + List < NodeInstance > found = []; + for (int i = 0; i < count; i++) { + NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus) + if (f) { + found << f + }; + } + return found + } + + @Test + public void testFind1NodeR0() throws Throwable { + NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) + log.info("found: $found") + assert [age3Active0].contains(found) + } + + @Test + public void testFind2NodeR0() throws Throwable { + NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) + log.info("found: $found") + assert [age2Active0, age3Active0].contains(found) + NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat) + log.info("found: $found2") + assert [age2Active0, age3Active0].contains(found2) + assert found != found2; + } + @Test + public void testFind3NodeR0ReturnsNull() throws Throwable { + assert 2== findNodes(2).size() + NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) + assert found == null; + } + + @Test + public void testFindNodesOneEntry() throws Throwable { + List<NodeInstance> nodes = findNodes(4, roleStat2) + assert 0 == nodes.size() + } + + @Test + public void testFindNodesIndependent() throws Throwable { + assert 2 == findNodes(2).size() + roleHistory.dump() + assert 0 == findNodes(3, roleStat2).size() + } + + @Test + public void testFindNodesFallsBackWhenUsed() throws Throwable { + // mark age2 and active 0 as busy, expect a null back + age2Active0.get(0).onStartCompleted() + assert age2Active0.getActiveRoleInstances(0) != 0 + age3Active0.get(0).onStartCompleted() + assert age3Active0.getActiveRoleInstances(0) != 0 + NodeInstance found = roleHistory.findNodeForNewInstance(roleStat) + log.info(found ?.toFullString()) + assert found == null + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy new file mode 100644 index 0000000..64db899 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.state.NodeInstance +import org.apache.slider.server.appmaster.state.NodeMap +import org.junit.Before +import org.junit.Test + +@Slf4j +@CompileStatic +class TestRoleHistoryFindNodesForRelease extends BaseMockAppStateTest { + + + @Override + String getTestName() { + return "TestFindNodesForRelease" + } + NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0) + NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0) + NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0) + NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0) + NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) + + List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0] + NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT); + + + @Before + public void setupNodeMap() { + nodeMap.insert(nodes) + } + + private void assertReleased( + int count, + List<NodeInstance> expected, + int role = 0) { + List<NodeInstance> released = nodeMap.findNodesForRelease(role, count) + assertListEquals(released, expected) + } + private void assertReleased( + List<NodeInstance> expected, + int role = 0) { + List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size()) + assertListEquals(released, expected) + } + + @Test + public void testListActiveNodes() throws Throwable { + assertListEquals(nodeMap.listActiveNodes(0), + [age1Active4,age2Active2, age4Active1]) + } + + @Test + public void testReleaseMinus1() throws Throwable { + try { + nodeMap.findNodesForRelease(0, -1) + fail("Expected an exception") + } catch (IllegalArgumentException e) { + } + } + @Test + public void testReleaseO() throws Throwable { + assertReleased(0, []) + } + + @Test + public void testRelease1() throws Throwable { + assertReleased(1, [age1Active4]) + } + + @Test + public void testRelease2() throws Throwable { + assertReleased(2, [age1Active4, age1Active4]) + } + + @Test + public void testRelease3() throws Throwable { + assertReleased(3, [age1Active4, age1Active4, age1Active4 ]) + } + + @Test + public void testRelease4() throws Throwable { + assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2]) + } + + @Test + public void testRelease5() throws Throwable { + assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1]) + } + + @Test + public void testRelease6() throws Throwable { + assertReleased( + [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4]) + } + + @Test + public void testRelease7() throws Throwable { + assertReleased( + [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, + age1Active4, age2Active2]) + } + + @Test + public void testRelease8() throws Throwable { + assertReleased(8, + [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, + age1Active4, age2Active2]) + } + + @Test + public void testPurgeInactiveTime3() throws Throwable { + assert nodeMap.purgeUnusedEntries(3) == 0; + } + + @Test + public void testPurgeInactiveTime4() throws Throwable { + assert nodeMap.purgeUnusedEntries(4) == 1; + } + @Test + public void testPurgeInactiveTime5() throws Throwable { + assert nodeMap.purgeUnusedEntries(5) == 1; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy new file mode 100644 index 0000000..612cce8 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.state.NodeInstance +import org.junit.Test + +/** + * Unit test to verify the comparators sort as expected + */ +class TestRoleHistoryNIComparators extends BaseMockAppStateTest { + + NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0) + NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0) + NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0) + NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0) + NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT) + + List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0] + + @Override + String getTestName() { + return "TestNIComparators" + } + + @Test + public void testNewerThan() throws Throwable { + + Collections.sort(nodes, new NodeInstance.newerThan(0)) + assertListEquals(nodes, + [age4Active1, age3Active0, age2Active2, age1Active4]) + } + + @Test + public void testNewerThanNoRole() throws Throwable { + + nodes << empty + Collections.sort(nodes, new NodeInstance.newerThan(0)) + assertListEquals(nodes, + [age4Active1, age3Active0, age2Active2, age1Active4, empty]) + } + + @Test + public void testMoreActiveThan() throws Throwable { + + Collections.sort(nodes, new NodeInstance.moreActiveThan(0)) + assertListEquals(nodes, + [age1Active4, age2Active2, age4Active1, age3Active0],) + } + + @Test + public void testMoreActiveThanEmpty() throws Throwable { + nodes << empty + Collections.sort(nodes, new NodeInstance.moreActiveThan(0)) + assertListEquals(nodes, + [age1Active4, age2Active2, age4Active1, age3Active0, empty]) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy new file mode 100644 index 0000000..7085678 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.state.NodeInstance +import org.apache.slider.server.appmaster.state.OutstandingRequest +import org.apache.slider.server.appmaster.state.OutstandingRequestTracker +import org.junit.Test + +class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest { + + NodeInstance host1 = new NodeInstance("host1", 3) + NodeInstance host2 = new NodeInstance("host2", 3) + + OutstandingRequestTracker tracker = new OutstandingRequestTracker() + + @Override + String getTestName() { + return "TestOutstandingRequestTracker" + } + + @Test + public void testAddRetrieveEntry() throws Throwable { + OutstandingRequest request = tracker.addRequest(host1, 0) + assert tracker.lookup(0, "host1").equals(request) + assert tracker.remove(request).equals(request) + assert !tracker.lookup(0, "host1") + } + + @Test + public void testAddCompleteEntry() throws Throwable { + tracker.addRequest(host1, 0) + tracker.addRequest(host2, 0) + tracker.addRequest(host1, 1) + assert tracker.onContainerAllocated(1, "host1") + assert !tracker.lookup(1, "host1") + assert tracker.lookup(0, "host1") + } + + @Test + public void testCancelEntries() throws Throwable { + OutstandingRequest r1 = tracker.addRequest(host1, 0) + OutstandingRequest r2 = tracker.addRequest(host2, 0) + OutstandingRequest r3 = tracker.addRequest(host1, 1) + List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0) + assert canceled.size() == 2 + assert canceled.contains(host1) + assert canceled.contains(host2) + assert tracker.lookup(1, "host1") + assert !tracker.lookup(0, "host1") + canceled = tracker.cancelOutstandingRequests(0) + assert canceled.size() == 0 + assert tracker.cancelOutstandingRequests(1).size() == 1 + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy new file mode 100644 index 0000000..0f4a4ca --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.FSDataOutputStream +import org.apache.hadoop.fs.Path +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.state.NodeEntry +import org.apache.slider.server.appmaster.state.NodeInstance +import org.apache.slider.server.appmaster.state.RoleHistory +import org.apache.slider.server.avro.RoleHistoryWriter +import org.junit.Test + +@Slf4j +@CompileStatic +class TestRoleHistoryRW extends BaseMockAppStateTest { + + static long time = System.currentTimeMillis(); + + @Override + String getTestName() { + return "TestHistoryRW" + } + + @Test + public void testWriteReadEmpty() throws Throwable { + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + roleHistory.onStart(fs, historyPath) + Path history = roleHistory.saveHistory(time++) + assert fs.isFile(history) + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + historyWriter.read(fs, history, roleHistory) + } + + @Test + public void testWriteReadData() throws Throwable { + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + assert !roleHistory.onStart(fs, historyPath) + String addr = "localhost" + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) + NodeEntry ne1 = instance.getOrCreate(0) + ne1.lastUsed = 0xf00d + + Path history = roleHistory.saveHistory(time++) + assert fs.isFile(history) + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) + + assert 0 < historyWriter.read(fs, history, rh2) + NodeInstance ni2 = rh2.getExistingNodeInstance(addr) + assert ni2 != null + NodeEntry ne2 = ni2.get(0) + assert ne2 !=null + assert ne2.lastUsed == ne1.lastUsed + } + + @Test + public void testWriteReadActiveData() throws Throwable { + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + roleHistory.onStart(fs, historyPath) + String addr = "localhost" + String addr2 = "rack1server5" + NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr) + NodeEntry orig1 = localhost.getOrCreate(0) + orig1.lastUsed = 0x10 + NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2) + NodeEntry orig2 = rack1server5.getOrCreate(1) + orig2.live = 3 + assert !orig2.available + NodeEntry orig3 = localhost.getOrCreate(1) + orig3.lastUsed = 0x20 + orig3.live = 1 + assert !orig3.available + orig3.release() + assert orig3.available + roleHistory.dump() + + long savetime = 0x0001000 + Path history = roleHistory.saveHistory(savetime) + assert fs.isFile(history) + describe("Loaded") + log.info("testWriteReadActiveData in $history") + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) + + assert 3 == historyWriter.read(fs, history, rh2) + rh2.dump() + + assert rh2.clusterSize == 2; + NodeInstance ni2 = rh2.getExistingNodeInstance(addr) + assert ni2 != null + NodeEntry loadedNE = ni2.get(0) + assert loadedNE.lastUsed == orig1.lastUsed + NodeInstance ni2b = rh2.getExistingNodeInstance(addr2) + assert ni2b != null + NodeEntry loadedNE2 = ni2b.get(1) + assert loadedNE2 != null + assert loadedNE2.lastUsed == savetime + assert rh2.thawedDataTime == savetime + + // now thaw it + rh2.buildAvailableNodeLists(); + describe("thawing") + rh2.dump(); + List<NodeInstance> available0 = rh2.cloneAvailableList(0) + assert available0.size() == 1 + + NodeInstance entry = available0.get(0) + assert entry.hostname == "localhost" + assert entry == localhost + List<NodeInstance> available1 = rh2.cloneAvailableList(1) + assert available1.size() == 2 + //and verify that even if last used was set, the save time is picked up + assert entry.get(1).lastUsed == roleHistory.saveTime + + } + + @Test + public void testWriteThaw() throws Throwable { + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + assert !roleHistory.onStart(fs, historyPath) + String addr = "localhost" + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) + NodeEntry ne1 = instance.getOrCreate(0) + ne1.lastUsed = 0xf00d + + Path history = roleHistory.saveHistory(time++) + long savetime =roleHistory.saveTime; + assert fs.isFile(history) + RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) + assert rh2.onStart(fs, historyPath) + NodeInstance ni2 = rh2.getExistingNodeInstance(addr) + assert ni2 != null + NodeEntry ne2 = ni2.get(0) + assert ne2 != null + assert ne2.lastUsed == ne1.lastUsed + assert rh2.thawedDataTime == savetime + } + + + @Test + public void testPurgeOlderEntries() throws Throwable { + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + time = 1; + Path file1 = touch(historyWriter, time++) + Path file2 = touch(historyWriter, time++) + Path file3 = touch(historyWriter, time++) + Path file4 = touch(historyWriter, time++) + Path file5 = touch(historyWriter, time++) + Path file6 = touch(historyWriter, time++) + + assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0 + assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1 + assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0 + assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3 + assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1 + try { + // make an impossible assertion that will fail if the method + // actually completes + assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1) + } catch (FileNotFoundException ignored) { + // expected + } + + } + + public Path touch(RoleHistoryWriter historyWriter, long time){ + Path path = historyWriter.createHistoryFilename(historyPath, time); + FSDataOutputStream out = fs.create(path); + out.close() + return path + } + + @Test + public void testSkipEmptyFileOnRead() throws Throwable { + describe "verify that empty histories are skipped on read; old histories purged" + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + roleHistory.onStart(fs, historyPath) + time = 0 + Path oldhistory = roleHistory.saveHistory(time++) + + String addr = "localhost" + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) + NodeEntry ne1 = instance.getOrCreate(0) + ne1.lastUsed = 0xf00d + + Path goodhistory = roleHistory.saveHistory(time++) + + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + Path touched = touch(historyWriter, time++) + + RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) + assert rh2.onStart(fs, historyPath) + NodeInstance ni2 = rh2.getExistingNodeInstance(addr) + assert ni2 != null + + //and assert the older file got purged + assert !fs.exists(oldhistory) + assert fs.exists(goodhistory) + assert fs.exists(touched ) + } + + @Test + public void testSkipBrokenFileOnRead() throws Throwable { + describe "verify that empty histories are skipped on read; old histories purged" + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + roleHistory.onStart(fs, historyPath) + time = 0 + Path oldhistory = roleHistory.saveHistory(time++) + + String addr = "localhost" + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) + NodeEntry ne1 = instance.getOrCreate(0) + ne1.lastUsed = 0xf00d + + Path goodhistory = roleHistory.saveHistory(time++) + + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + Path badfile = historyWriter.createHistoryFilename(historyPath, time++) + FSDataOutputStream out = fs.create(badfile) + out.writeBytes("{broken:true}") + out.close() + + RoleHistory rh2 = new RoleHistory(MockFactory.ROLES) + describe("IGNORE STACK TRACE BELOW") + + assert rh2.onStart(fs, historyPath) + + describe( "IGNORE STACK TRACE ABOVE") + NodeInstance ni2 = rh2.getExistingNodeInstance(addr) + assert ni2 != null + + //and assert the older file got purged + assert !fs.exists(oldhistory) + assert fs.exists(goodhistory) + assert fs.exists(badfile ) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy new file mode 100644 index 0000000..a0663e8 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history + +import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.Path +import org.apache.slider.common.SliderKeys +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockFactory +import org.apache.slider.server.appmaster.state.NodeEntry +import org.apache.slider.server.appmaster.state.NodeInstance +import org.apache.slider.server.appmaster.state.RoleHistory +import org.apache.slider.server.avro.RoleHistoryWriter +import org.junit.Test + +import java.util.regex.Matcher +import java.util.regex.Pattern + +@Slf4j +class TestRoleHistoryRWOrdering extends BaseMockAppStateTest { + + def paths = pathlist( + [ + "hdfs://localhost/history-0406c.json", + "hdfs://localhost/history-5fffa.json", + "hdfs://localhost/history-0001a.json", + "hdfs://localhost/history-0001f.json", + ] + ) + Path h_0406c = paths[0] + Path h_5fffa = paths[1] + Path h_0001a = paths[3] + + + List<Path> pathlist(List<String> pathnames) { + def result = [] + pathnames.each { result << new Path(new URI(it as String)) } + result + } + + @Override + String getTestName() { + return "TestHistoryRWOrdering" + } + + + /** + * This tests regexp pattern matching. It uses the current time so isn't + * repeatable -but it does test a wider range of values in the process + * @throws Throwable + */ + @Test + public void testPatternRoundTrip() throws Throwable { + describe "test pattern matching of names" + long value=System.currentTimeMillis() + String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value) + String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN + Pattern pattern = Pattern.compile(matchpattern) + Matcher matcher = pattern.matcher(name); + if (!matcher.find()) { + throw new Exception("No match for pattern $matchpattern in $name") + } + } + + + @Test + public void testWriteSequenceReadData() throws Throwable { + describe "test that if multiple entries are written, the newest is picked up" + long time = System.currentTimeMillis(); + + RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES) + assert !roleHistory.onStart(fs, historyPath) + String addr = "localhost" + NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr) + NodeEntry ne1 = instance.getOrCreate(0) + ne1.lastUsed = 0xf00d + + Path history1 = roleHistory.saveHistory(time++) + Path history2 = roleHistory.saveHistory(time++) + Path history3 = roleHistory.saveHistory(time++) + + //inject a later file with a different name + sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world") + + + RoleHistoryWriter historyWriter = new RoleHistoryWriter(); + + List<Path> entries = historyWriter.findAllHistoryEntries( + fs, + historyPath, + false) + assert entries.size() == 3 + assert entries[0] == history3 + assert entries[1] == history2 + assert entries[2] == history1 + } + + @Test + public void testPathStructure() throws Throwable { + assert h_5fffa.getName() == "history-5fffa.json" + } + + @Test + public void testPathnameComparator() throws Throwable { + + def newerName = new RoleHistoryWriter.NewerFilesFirst() + + log.info("$h_5fffa name is ${h_5fffa.getName()}") + log.info("$h_0406c name is ${h_0406c.getName()}") + assert newerName.compare(h_5fffa, h_5fffa) == 0 + assert newerName.compare(h_5fffa, h_0406c) < 0 + assert newerName.compare(h_5fffa, h_0001a) < 0 + assert newerName.compare(h_0001a, h_5fffa) > 0 + + } + + @Test + public void testPathSort() throws Throwable { + def paths2 = new ArrayList(paths) + RoleHistoryWriter.sortHistoryPaths(paths2) + assertListEquals(paths2, + [ + paths[1], + paths[0], + paths[3], + paths[2] + ]) + } +}
