Wait for the HA state to stabilize before letting requests in There is a period of time when the state is MASTER, but the context is not fully initialized.
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/f2460a69 Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/f2460a69 Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/f2460a69 Branch: refs/heads/master Commit: f2460a6971f3328ebfb4410399ed9b1546f4d062 Parents: 3b99a96 Author: Svetoslav Neykov <[email protected]> Authored: Thu Mar 5 15:06:20 2015 +0200 Committer: Svetoslav Neykov <[email protected]> Committed: Thu Mar 19 16:01:29 2015 +0200 ---------------------------------------------------------------------- .../management/ha/HighAvailabilityManager.java | 5 ++++- .../management/ha/HighAvailabilityManagerImpl.java | 9 +++++++++ .../internal/NonDeploymentManagementContext.java | 4 ++++ .../rest/filter/HaStateCheckResourceFilter.java | 15 +++++++++++++-- 4 files changed, 30 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/f2460a69/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java ---------------------------------------------------------------------- diff --git a/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java b/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java index 04079bb..2a1af9a 100644 --- a/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java +++ b/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java @@ -44,6 +44,9 @@ public interface HighAvailabilityManager { ManagementNodeState getNodeState(); + /** The time in milliseconds when the state was last changed */ + long getLastStateChange(); + /** * @param persister * @return self @@ -128,5 +131,5 @@ public interface HighAvailabilityManager { /** Returns a collection of metrics */ Map<String,Object> getMetrics(); - + } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/f2460a69/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java index bc5c398..b2af52a 100644 --- a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java +++ b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java @@ -1077,4 +1077,13 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager { return result; } + @Override + public long getLastStateChange() { + if (nodeStateHistory.size() > 0) { + return (Long)nodeStateHistory.get(0).get("timestamp"); + } else { + return 0; + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/f2460a69/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java b/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java index ea4ec2b..0972871 100644 --- a/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java +++ b/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java @@ -603,5 +603,9 @@ public class NonDeploymentManagementContext implements ManagementContextInternal public void publishClearNonMaster() { throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+" is not valid for this operation."); } + @Override + public long getLastStateChange() { + throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+" is not valid for this operation."); + } } } http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/f2460a69/usage/rest-server/src/main/java/brooklyn/rest/filter/HaStateCheckResourceFilter.java ---------------------------------------------------------------------- diff --git a/usage/rest-server/src/main/java/brooklyn/rest/filter/HaStateCheckResourceFilter.java b/usage/rest-server/src/main/java/brooklyn/rest/filter/HaStateCheckResourceFilter.java index 9da0766..d894252 100644 --- a/usage/rest-server/src/main/java/brooklyn/rest/filter/HaStateCheckResourceFilter.java +++ b/usage/rest-server/src/main/java/brooklyn/rest/filter/HaStateCheckResourceFilter.java @@ -32,6 +32,7 @@ import brooklyn.config.BrooklynServiceAttributes; import brooklyn.entity.rebind.RebindManagerImpl.RebindTracker; import brooklyn.management.ManagementContext; import brooklyn.management.ha.ManagementNodeState; +import brooklyn.util.time.Duration; import com.google.common.collect.ImmutableSet; import com.sun.jersey.api.model.AbstractMethod; @@ -44,11 +45,13 @@ import com.sun.jersey.spi.container.ResourceFilterFactory; public class HaStateCheckResourceFilter implements ResourceFilterFactory { private static final Set<ManagementNodeState> HOT_STATES = ImmutableSet.of( ManagementNodeState.MASTER, ManagementNodeState.HOT_STANDBY, ManagementNodeState.HOT_BACKUP); + private static final long STATE_CHANGE_SETTLE_OFFSET = Duration.seconds(10).toMilliseconds(); @Context - private ServletContext servletContext; + private ManagementContext mgmt; private static class MethodFilter implements ResourceFilter, ContainerRequestFilter { + private AbstractMethod am; private ManagementContext mgmt; @@ -80,7 +83,15 @@ public class HaStateCheckResourceFilter implements ResourceFilterFactory { } private boolean isStateLoaded() { - return isHaHotStatus() && !RebindTracker.isRebinding(); + return isHaHotStatus() && !RebindTracker.isRebinding() && !recentlySwitchedState(); + } + + // Ideally there will be a separate state to indicate that we switched state + // but still haven't finished rebinding. There's a gap between changing the state + // and starting rebind so add a time offset just to be sure. + private boolean recentlySwitchedState() { + long lastStateChange = mgmt.getHighAvailabilityManager().getLastStateChange(); + return System.currentTimeMillis() - lastStateChange < STATE_CHANGE_SETTLE_OFFSET; } private boolean isUnsafe(ContainerRequest request) {
