fixing issues faced while evaluating status
Project: http://git-wip-us.apache.org/repos/asf/stratos/repo Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/932f31f2 Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/932f31f2 Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/932f31f2 Branch: refs/heads/4.0.0-grouping Commit: 932f31f2bce87c3b48c9bf373c744b5ec1f029c5 Parents: 605b92f Author: reka <[email protected]> Authored: Thu Oct 30 15:31:27 2014 +0530 Committer: reka <[email protected]> Committed: Thu Oct 30 15:50:19 2014 +0530 ---------------------------------------------------------------------- .../monitor/AbstractClusterMonitor.java | 5 +- .../monitor/ParentComponentMonitor.java | 78 ++++++++++++-------- .../monitor/application/ApplicationMonitor.java | 11 ++- .../monitor/cluster/ClusterMonitor.java | 29 ++++---- .../autoscaler/monitor/group/GroupMonitor.java | 19 ++++- .../status/checker/StatusChecker.java | 24 +++--- 6 files changed, 103 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/stratos/blob/932f31f2/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java index fb23985..881f838 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java @@ -240,12 +240,13 @@ abstract public class AbstractClusterMonitor extends Monitor implements Runnable log.info("[Cluster] " + clusterId + "is not notifying the parent, " + "since it is identified as the independent unit"); - } else if (status == ClusterStatus.Terminating) { + /*} else if (status == ClusterStatus.Terminating) { // notify parent log.info("[Cluster] " + clusterId + " is not notifying the parent, " + "since it is in Terminating State"); - +*/ } else { + log.info("[Group] " + this.id + "is notifying the [parent] " + this.parent.getId()); MonitorStatusEventBuilder.handleClusterStatusEvent(this.parent, this.status, this.clusterId); } //} http://git-wip-us.apache.org/repos/asf/stratos/blob/932f31f2/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ParentComponentMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ParentComponentMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ParentComponentMonitor.java index 537e855..58cdb1a 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ParentComponentMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ParentComponentMonitor.java @@ -30,6 +30,7 @@ import org.apache.stratos.autoscaler.grouping.dependency.context.ApplicationCont import org.apache.stratos.autoscaler.grouping.topic.StatusEventPublisher; import org.apache.stratos.autoscaler.monitor.events.MonitorStatusEvent; import org.apache.stratos.autoscaler.status.checker.StatusChecker; +import org.apache.stratos.messaging.domain.topology.Application; import org.apache.stratos.messaging.domain.topology.ApplicationStatus; import org.apache.stratos.messaging.domain.topology.GroupStatus; import org.apache.stratos.messaging.domain.topology.ParentComponent; @@ -190,42 +191,17 @@ public abstract class ParentComponentMonitor extends Monitor { * Make sure that all the dependents have been terminated properly to start the recovery */ if (terminationList != null) { - for (ApplicationContext context1 : terminationList) { - if (this.aliasToActiveMonitorsMap.containsKey(context1.getId())) { - log.warn("Dependent [monitor] " + context1.getId() + " not in the correct state"); - allDependentTerminated = false; - } else if (this.aliasToInActiveMonitorsMap.containsKey(context1.getId())) { - log.info("Waiting for the [dependent] " + context1.getId() + " to be terminated..."); - allDependentTerminated = false; - } else { - allDependentTerminated = true; - } - } + allDependentTerminated = allDependentTerminated(terminationList); } List<ApplicationContext> parentContexts = this.dependencyTree.findAllParentContextWithId(idOfEvent); - boolean canStart = false; + boolean parentsTerminated = false; if (parentContexts != null) { - for (ApplicationContext context1 : parentContexts) { - if (this.aliasToInActiveMonitorsMap.containsKey(context1.getId())) { - log.info("Waiting for the [Parent Monitor] " + context1.getId() - + " to be terminated"); - canStart = false; - } else if (this.aliasToActiveMonitorsMap.containsKey(context1.getId())) { - if (canStart) { - log.warn("Found the Dependent [monitor] " + context1.getId() - + " in the active list wrong state"); - } - } else { - log.info("[Parent Monitor] " + context1.getId() - + " has already been terminated"); - canStart = true; - } - } + parentsTerminated = allParentTerminated(parentContexts); } if ((terminationList != null && allDependentTerminated || terminationList == null) && - (parentContexts != null && canStart || parentContexts == null)) { + (parentContexts != null && parentsTerminated || parentContexts == null)) { //Find the non existent monitor by traversing dependency tree try { this.startDependencyOnTermination(); @@ -235,13 +211,53 @@ public abstract class ParentComponentMonitor extends Monitor { } } else { StatusChecker.getInstance().onChildStatusChange(idOfEvent, this.id, this.appId); - log.info("Waiting for the dependent of [monitor] " + idOfEvent - + " to be terminated"); + log.info("" + + "Checking the status of group/application as no dependent found..."); } } + private boolean allDependentTerminated(List<ApplicationContext> terminationList) { + boolean allDependentTerminated = false; + for (ApplicationContext context1 : terminationList) { + if (this.aliasToActiveMonitorsMap.containsKey(context1.getId())) { + log.warn("Dependent [monitor] " + context1.getId() + " not in the correct state"); + allDependentTerminated = false; + return allDependentTerminated; + } else if (this.aliasToInActiveMonitorsMap.containsKey(context1.getId())) { + log.info("Waiting for the [dependent] " + context1.getId() + " to be terminated..."); + allDependentTerminated = false; + return allDependentTerminated; + } else { + allDependentTerminated = true; + } + } + return allDependentTerminated; + } + + + private boolean allParentTerminated(List<ApplicationContext> parentContexts) { + boolean parentsTerminated = false; + for (ApplicationContext context1 : parentContexts) { + if (this.aliasToInActiveMonitorsMap.containsKey(context1.getId())) { + log.info("Waiting for the [Parent Monitor] " + context1.getId() + + " to be terminated"); + parentsTerminated = false; + return parentsTerminated; + } else if (this.aliasToActiveMonitorsMap.containsKey(context1.getId())) { + if (parentsTerminated) { + log.warn("Found the Dependent [monitor] " + context1.getId() + + " in the active list wrong state"); + } + } else { + log.info("[Parent Monitor] " + context1.getId() + + " has already been terminated"); + parentsTerminated = true; + } + } + return parentsTerminated; + } /** * This will start the parallel dependencies at once from the top level. http://git-wip-us.apache.org/repos/asf/stratos/blob/932f31f2/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/application/ApplicationMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/application/ApplicationMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/application/ApplicationMonitor.java index e7e3238..6256c02 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/application/ApplicationMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/application/ApplicationMonitor.java @@ -162,9 +162,18 @@ public class ApplicationMonitor extends ParentComponentMonitor { } else if (status1 == ClusterStatus.Inactive || status1 == GroupStatus.Inactive) { onChildInActiveEvent(id); + } else if (status1 == ClusterStatus.Terminating || status1 == GroupStatus.Terminated) { + //mark the child monitor as inActive in the map + this.markMonitorAsInactive(id); + } else if (status1 == ClusterStatus.Terminated || status1 == GroupStatus.Terminated) { //Check whether all dependent goes Terminated and then start them in parallel. - this.aliasToInActiveMonitorsMap.remove(id); + if(this.aliasToInActiveMonitorsMap.containsKey(id)) { + this.aliasToInActiveMonitorsMap.remove(id); + } else { + log.warn("[monitor] " + id + " cannot be found in the inActive monitors list"); + } + if (this.status != ApplicationStatus.Terminating || this.status != ApplicationStatus.Terminated) { onChildTerminatedEvent(id); } else { http://git-wip-us.apache.org/repos/asf/stratos/blob/932f31f2/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java index 6d7e8ca..67539ed 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/cluster/ClusterMonitor.java @@ -73,18 +73,26 @@ public class ClusterMonitor extends AbstractClusterMonitor { //status = ClusterStatus.Created; } + private static void terminateMember(String memberId) { + try { + CloudControllerClient.getInstance().terminate(memberId); + + } catch (TerminationException e) { + log.error("Unable to terminate member [member id ] " + memberId, e); + } + } @Override public void run() { while (!isDestroyed()) { try { - if ((this.status.getCode() <= ClusterStatus.Active.getCode()) || - (this.status == ClusterStatus.Inactive && !hasDependent) || - !this.hasFaultyMember) { + if (((this.status.getCode() <= ClusterStatus.Active.getCode()) || + (this.status == ClusterStatus.Inactive && !hasDependent)) && !this.hasFaultyMember) { if (log.isDebugEnabled()) { log.debug("Cluster monitor is running.. " + this.toString()); } monitor(); + } else { if (log.isDebugEnabled()) { log.debug("Cluster monitor is suspended as the cluster is in " + @@ -106,8 +114,8 @@ public class ClusterMonitor extends AbstractClusterMonitor { @Override public void terminateAllMembers() { - Thread memberTerminator = new Thread(new Runnable(){ - public void run(){ + Thread memberTerminator = new Thread(new Runnable() { + public void run() { for (NetworkPartitionContext networkPartitionContext : networkPartitionCtxts.values()) { for (PartitionContext partitionContext : networkPartitionContext.getPartitionCtxts().values()) { @@ -125,7 +133,7 @@ public class ClusterMonitor extends AbstractClusterMonitor { } // pending members - for (MemberContext pendingMemberCtxt : partitionContext.getPendingMembers()) { + for (MemberContext pendingMemberCtxt : partitionContext.getPendingMembers()) { log.info("Terminating pending member [member id] " + pendingMemberCtxt.getMemberId()); terminateMember(pendingMemberCtxt.getMemberId()); } @@ -146,15 +154,6 @@ public class ClusterMonitor extends AbstractClusterMonitor { memberTerminator.start(); } - private static void terminateMember (String memberId) { - try { - CloudControllerClient.getInstance().terminate(memberId); - - } catch (TerminationException e) { - log.error("Unable to terminate member [member id ] " + memberId, e); - } - } - private boolean isPrimaryMember(MemberContext memberContext) { Properties props = memberContext.getProperties(); if (log.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/stratos/blob/932f31f2/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/group/GroupMonitor.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/group/GroupMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/group/GroupMonitor.java index fc79c59..a9aeb57 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/group/GroupMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/group/GroupMonitor.java @@ -24,6 +24,7 @@ import org.apache.stratos.autoscaler.exception.DependencyBuilderException; import org.apache.stratos.autoscaler.exception.TopologyInConsistentException; import org.apache.stratos.autoscaler.grouping.topic.StatusEventPublisher; import org.apache.stratos.autoscaler.monitor.EventHandler; +import org.apache.stratos.autoscaler.monitor.Monitor; import org.apache.stratos.autoscaler.monitor.MonitorStatusEventBuilder; import org.apache.stratos.autoscaler.monitor.ParentComponentMonitor; import org.apache.stratos.autoscaler.monitor.events.GroupStatusEvent; @@ -72,15 +73,25 @@ public class GroupMonitor extends ParentComponentMonitor implements EventHandler } else if (status1 == ClusterStatus.Inactive || status1 == GroupStatus.Inactive) { onChildInActiveEvent(id); + } else if (status1 == ClusterStatus.Terminating || status1 == GroupStatus.Terminated) { + //mark the child monitor as inActive in the map + this.markMonitorAsInactive(id); + } else if (status1 == ClusterStatus.Terminated || status1 == GroupStatus.Terminated) { //Check whether all dependent goes Terminated and then start them in parallel. - this.aliasToInActiveMonitorsMap.remove(id); + if(this.aliasToInActiveMonitorsMap.containsKey(id)) { + this.aliasToInActiveMonitorsMap.remove(id); + } else { + log.warn("[monitor] " + id + " cannot be found in the inActive monitors list"); + } + if (this.status != GroupStatus.Terminating || this.status != GroupStatus.Terminated) { onChildTerminatedEvent(id); } else { StatusChecker.getInstance().onChildStatusChange(id, this.id, this.appId); log.info("Executing the un-subscription request for the [monitor] " + id); } + } } @@ -90,7 +101,6 @@ public class GroupMonitor extends ParentComponentMonitor implements EventHandler if (statusEvent.getStatus() == GroupStatus.Terminating || statusEvent.getStatus() == ApplicationStatus.Terminating) { StatusEventPublisher.sendGroupTerminatingEvent(appId, id); - this.markMonitorAsInactive(id); } } @@ -147,12 +157,13 @@ public class GroupMonitor extends ParentComponentMonitor implements EventHandler log.info("[Group] " + this.id + "is not notifying the parent, " + "since it is identified as the independent unit"); - } else if (status == GroupStatus.Terminating) { + /*} else if (status == GroupStatus.Terminating) { log.info("[Group] " + this.id + " is not notifying the parent, " + "since it is in Terminating State"); - +*/ } else { // notify parent + log.info("[Group] " + this.id + "is notifying the [parent] " + this.parent.getId()); MonitorStatusEventBuilder.handleGroupStatusEvent(this.parent, this.status, this.id); } //} http://git-wip-us.apache.org/repos/asf/stratos/blob/932f31f2/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/status/checker/StatusChecker.java ---------------------------------------------------------------------- diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/status/checker/StatusChecker.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/status/checker/StatusChecker.java index 3a925d1..bc08c56 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/status/checker/StatusChecker.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/status/checker/StatusChecker.java @@ -68,6 +68,7 @@ public class StatusChecker { // if active then notify upper layer if (clusterActive) { //send event to cluster status topic + monitor.setHasFaultyMember(false); StatusEventPublisher.sendClusterActivatedEvent(monitor.getAppId(), monitor.getServiceId(), monitor.getClusterId()); } @@ -251,7 +252,10 @@ public class StatusChecker { */ private boolean updateChildStatus(String appId, String id, Map<String, Group> groups, Map<String, ClusterDataHolder> clusterData, ParentComponent parent) { - boolean groupActive = false; + boolean groupsActive = false; + boolean groupsTerminated = false; + boolean groupsInActive = false; + boolean groupsTerminating = false; ClusterStatus clusterStatus; GroupStatus groupStatus; boolean childFound = false; @@ -350,19 +354,19 @@ public class StatusChecker { private GroupStatus getGroupStatus(Map<String, Group> groups) { GroupStatus status = null; - boolean groupActive = false; - boolean groupTerminated = false; + boolean groupActive = true; + boolean groupTerminated = true; for (Group group : groups.values()) { if (group.getStatus() == GroupStatus.Active) { - groupActive = true; - groupTerminated = false; + groupActive = groupActive && true; + groupTerminated = false; } else if (group.getStatus() == GroupStatus.Inactive) { status = GroupStatus.Inactive; break; } else if (group.getStatus() == GroupStatus.Terminated) { groupActive = false; - groupTerminated = true; + groupTerminated = groupTerminated && true; } else if (group.getStatus() == GroupStatus.Created) { groupActive = false; groupTerminated = false; @@ -386,13 +390,13 @@ public class StatusChecker { private ClusterStatus getClusterStatus(Map<String, ClusterDataHolder> clusterData) { ClusterStatus status = null; - boolean clusterActive = false; - boolean clusterTerminated = false; + boolean clusterActive = true; + boolean clusterTerminated = true; for (Map.Entry<String, ClusterDataHolder> clusterDataHolderEntry : clusterData.entrySet()) { Service service = TopologyManager.getTopology().getService(clusterDataHolderEntry.getValue().getServiceType()); Cluster cluster = service.getCluster(clusterDataHolderEntry.getValue().getClusterId()); if (cluster.getStatus() == ClusterStatus.Active) { - clusterActive = true; + clusterActive = clusterActive && true; clusterTerminated = false; } else if (cluster.getStatus() == ClusterStatus.Inactive) { status = ClusterStatus.Inactive; @@ -401,7 +405,7 @@ public class StatusChecker { break; } else if (cluster.getStatus() == ClusterStatus.Terminated) { clusterActive = false; - clusterTerminated = true; + clusterTerminated = clusterTerminated && true; } else if (cluster.getStatus() == ClusterStatus.Terminating) { status = ClusterStatus.Terminating; clusterActive = false;
