SLIDER-270 assertion checking has found the location of the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df963f24 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df963f24 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df963f24 Branch: refs/heads/feature/SLIDER-151_REST_API Commit: df963f24b474f33d00719f91d8b50e83a8f55b6f Parents: 291a82e Author: Steve Loughran <ste...@apache.org> Authored: Tue Aug 5 21:35:28 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Aug 5 21:35:54 2014 +0100 ---------------------------------------------------------------------- .../slider/server/appmaster/state/AppState.java | 25 ++++++++++++++------ .../server/appmaster/state/RoleHistory.java | 6 ++--- .../TestRoleHistoryContainerEvents.groovy | 2 +- .../src/test/resources/log4j.properties | 1 + 4 files changed, 23 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 74e91a0..2b17c3a 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -1172,7 +1172,7 @@ public class AppState { RoleInstance roleInstance; if (containersBeingReleased.containsKey(containerId)) { - log.info("Container was queued for release"); + log.info("Container was queued for release : {}", containerId); Container container = containersBeingReleased.remove(containerId); RoleStatus roleStatus = lookupRoleStatus(container); int releasing = roleStatus.decReleasing(); @@ -1183,7 +1183,7 @@ public class AppState { actual, releasing, completedCount); - roleHistory.onReleaseCompleted(container); + roleHistory.onReleaseCompleted(container, true); } else if (surplusNodes.remove(containerId)) { //its a surplus one being purged @@ -1251,18 +1251,28 @@ public class AppState { //record the complete node's details; this pulls it from the livenode set //remove the node ContainerId id = status.getContainerId(); + log.info("Removing node ID {}", id); RoleInstance node = getLiveNodes().remove(id); - if (node == null) { - log.warn("Received notification of completion of unknown node {}", id); - completionOfNodeNotInLiveListEvent.incrementAndGet(); - - } else { + if (node != null) { node.state = ClusterDescription.STATE_DESTROYED; node.exitCode = status.getExitStatus(); node.diagnostics = status.getDiagnostics(); getCompletedNodes().put(id, node); result.roleInstance = node; + } else { + // not in the list + log.warn("Received notification of completion of unknown node {}", id); + completionOfNodeNotInLiveListEvent.incrementAndGet(); + } + // finally, verify the node doesn't exist any more + assert !containersBeingReleased.containsKey( + containerId) : "container in release queue"; + assert !getLiveNodes().containsKey( + containerId) : " container in live nodes"; + assert getActiveContainer(containerId) == + null : "Container still in active container list"; + return result; } @@ -1422,6 +1432,7 @@ public class AppState { * @throws SliderInternalStateException if the operation reveals that * the internal state of the application is inconsistent. */ + @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") public List<AbstractRMOperation> reviewOneRole(RoleStatus role) throws SliderInternalStateException, TriggerClusterTeardownException { List<AbstractRMOperation> operations = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 0cd2b39..de7e9d8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -29,7 +29,6 @@ import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.avro.RoleHistoryHeader; import org.apache.slider.server.avro.RoleHistoryWriter; -import org.mortbay.log.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -659,10 +658,11 @@ public class RoleHistory { /** * App state notified of a container completed * @param container completed container + * @param wasReleased * @return true if the node was queued */ - public boolean onReleaseCompleted(Container container) { - return markContainerFinished(container, true, false); + public boolean onReleaseCompleted(Container container, boolean wasReleased) { + return markContainerFinished(container, wasReleased, false); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy index 795b48f..1a71aed 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy @@ -158,7 +158,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { assert roleEntry.active == 0 // release completed - roleHistory.onReleaseCompleted(container) + roleHistory.onReleaseCompleted(container, true) assert roleEntry.releasing == 0 assert roleEntry.live == 0 assert roleEntry.active == 0 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties index a552a55..1cc355a 100644 --- a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties +++ b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties @@ -57,3 +57,4 @@ log4j.logger.org.apache.hadoop.security.token.delegation=WARN log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN +log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager=WARN \ No newline at end of file
