Author: challngr
Date: Fri Feb 7 13:48:17 2014
New Revision: 1565655
URL: http://svn.apache.org/r1565655
Log:
UIMA-3405 Implement service instance failure/restart policy.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/AServicePing.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/AServicePing.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/AServicePing.java?rev=1565655&r1=1565654&r2=1565655&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/AServicePing.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/AServicePing.java
Fri Feb 7 13:48:17 2014
@@ -28,9 +28,21 @@ import org.apache.uima.ducc.common.IServ
public abstract class AServicePing
{
+ protected int[] failure_window = null; // tracks consecutive failures
within a window
+ protected int failure_cursor = 0; // cursor to track failures
within the current window
+ protected int total_failures = 0; // current total run failures.
usually monotonically increasing.
+ protected int failure_max = 3; // max allowed failures within
any given window
+
+ protected int failure_window_size = 30; // 30 minutes. overridden at
first ping
+ protected int monitor_rate = 1; // ping rate, in minutes, min
1 used for calculations
+
+ protected boolean log_enabled = false;
+ protected long service_id = 0;
protected Properties smState;
+ private org.apache.uima.ducc.common.utils.DuccLogger duccLogger = null;
+
/**
* Called by the ping driver, to pass in useful things the pinger may want.
* @param arguments This is passed in from the service specification's
@@ -42,6 +54,43 @@ public abstract class AServicePing
public abstract void init(String arguments, String endpoint) throws
Exception;
/**
+ * Called by the ping driver to initialize static information about the
service and
+ * pinger. This guy calls the public init() method and is not intended
for public
+ * consumption.
+ *
+ * This guy initializes the default failure monitor, service id, whether
the
+ * service log is enabled, and the monitor (ping) rate, all of which are
+ * free to be used by derived classes.
+ *
+ * @param arguments This is passed in from the service specification's
+ * service_ping_arguments string.
+ *
+ * @param endpoint This is the name of the service endpoint, as passed in
+ * at service registration.
+ *
+ * @param initProps Properties file with static data about the service and
+ * pinger.
+ */
+ public void init(String arguments, String endpoint, Properties initProps)
+ throws Exception
+ {
+ failure_window_size =
Integer.parseInt(initProps.getProperty("failure-window"));
+ failure_window = new int[failure_window_size];
+ failure_cursor = 0;
+
+ monitor_rate = Integer.parseInt(initProps.getProperty("monitor-rate")
) / 60000; // convert to minutes
+ if (monitor_rate <= 0 ) monitor_rate = 1;
// minimum 1 minute allowed
+
+ service_id = Long.parseLong(initProps.getProperty("service-id") );
+
+ log_enabled = Boolean.parseBoolean(initProps.getProperty("do-log"));
+
+ failure_max = Integer.parseInt(initProps.getProperty("failure-max"));
+
+ init(arguments, endpoint);
+ }
+
+ /**
* Stop is called by the ping wrapper when it is being killed.
Implementors may optionally
* override this method with conenction shutdown code.
*/
@@ -53,31 +102,135 @@ public abstract class AServicePing
* for correct management and display of the service.
*/
public abstract IServiceStatistics getStatistics();
-
+ /**
+ * Current state of the monitored service is passed in here.
+ */
public void setSmState(Properties props)
{
smState = props;
}
+ /**
+ * Getter of the service state; Implementors may just access it directly
if they want.
+ */
public Properties getSmState()
{
return smState;
}
+ /**
+ * Called by the service manager to query the number of additional needed
service instances.
+ */
public int getAdditions()
{
return 0;
}
+ /**
+ * Called by the service manager to query the number of service insances
to dump.
+ */
public int getDeletions()
{
return 0;
}
+ private String fmtArray(int[] array)
+ {
+ Object[] vals = new Object[array.length];
+ StringBuffer sb = new StringBuffer();
+
+ for ( int i = 0; i < array.length; i++ ) {
+ sb.append("%3s ");
+ vals[i] = Integer.toString(array[i]);
+ }
+ return String.format(sb.toString(), vals);
+ }
+
+ /**
+ * This is intended for use by the SM when it drives a pinger in an
internal thread. External
+ * pingers won't have this set.
+ *
+ * However, external pinger's stdout is picked up by DUCC so the logger
will still print
+ * stuff to the service log without the use of the ducc logger.
+ */
+ public void setLogger(org.apache.uima.ducc.common.utils.DuccLogger logger)
+ {
+ this.duccLogger = logger;
+ }
+
+ private void doLog(String methodName, Object ... msg)
+ {
+ if ( !log_enabled ) return;
+
+ StringBuffer buf = new StringBuffer(methodName);
+ buf.append(" ");
+ buf.append(Long.toString(service_id));
+ for ( Object o : msg ) {
+ buf.append(" ");
+ if ( o == null ) {
+ buf.append("<null>");
+ } else {
+ buf.append(o.toString());
+ }
+ }
+
+ if ( duccLogger != null ) {
+ duccLogger.info(methodName, null, buf);
+ } else {
+ System.out.println(buf);
+ }
+ }
+
+ /**
+ * This determines if there have been excessive service instance failures
by tracking the
+ * number of failures, not consecutive, but rather within a window of
time. It may be
+ * overridden by extending monitors.
+ */
public boolean isExcessiveFailures()
{
- return false;
+ String methodName = "isExcessiveFailures";
+ boolean excessive_failures = false;
+
+ // Calculate total instance failures within some configured window.
If we get a cluster
+ // of failures, signal excessive failures so SM stops spawning new
ones.
+ int failures = Integer.parseInt(smState.getProperty("run-failures"));
+ doLog(methodName, "failures:", failures, "total_failures",
total_failures);
+ if ( failures > 0 ) {
+ int diff = Math.max(0, failures - total_failures); // nfailures
since last update
+ if ( diff < 0 ) {
+ // This indicates an instance was restarted, which forces a
cleaning of
+ // failure conditions.
+ total_failures = 0;
+ failure_cursor = 0;
+ for ( int i = 0; i < failure_window_size; i++ ) {
+ failure_window[i] = 0;
+ }
+ } else if ( diff > 0 ) {
+ total_failures += diff;
+ }
+
+ if ( diff >= 0 ) {
+ failure_window[failure_cursor++] = diff;
+ }
+
+ doLog(methodName, "failures", failures, "total_failures",
total_failures,
+ "failure_window", fmtArray(failure_window),
"failure_cursor", failure_cursor);
+
+ failure_cursor = failure_cursor % failure_window_size;
+
+ int windowed_failures = 0;
+ excessive_failures = false;
+ for ( int i = 0; i < failure_window_size; i++ ) {
+ windowed_failures += failure_window[i];
+ }
+ if ( windowed_failures >= failure_max ) {
+ excessive_failures = true;
+ }
+ doLog(methodName, "windowed_failures", windowed_failures,
"excessive_failures", excessive_failures);
+ }
+ return excessive_failures;
}
+
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java?rev=1565655&r1=1565654&r2=1565655&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/UimaAsPing.java
Fri Feb 7 13:48:17 2014
@@ -45,18 +45,6 @@ public class UimaAsPing
{
String ep;
- int failure_max = 5; // max consecutive run failures before
reporting excessive failures
- // which prevents restart of instances
- int current_failures = 0; // current consecutive run failures
- int consecutive_failures = 0; // n failures in consecutive pings
- int failure_window_size = 15; // 15 minutes
- int monitor_rate = 1; // ping rate, in minutes, min 1 used for
calculations
- int fail_index = 0;
- int[] failure_window = null; // tracks consecutive failures within a
window
- int failure_cursor = 0;
- long service_id = 0;
-
- boolean excessive_failures = false;
String endpoint;
String broker;
@@ -73,7 +61,6 @@ public class UimaAsPing
String nodeIp;
String pid;
boolean gmfail = false;
- boolean enable_log = false;
public UimaAsPing()
{
@@ -128,15 +115,8 @@ public class UimaAsPing
}
meta_timeout = props.getIntProperty ("meta-timeout" ,
5000);
broker_jmx_port = props.getIntProperty ("broker-jmx-port",
1099);
- enable_log = props.getBooleanProperty("enable-log" ,
false);
- failure_max = props.getIntProperty ("max-failures" ,
failure_max);
- failure_window_size = props.getIntProperty ("failure-window" ,
failure_window_size);
- failure_window = new int[failure_window_size];
- failure_cursor = 0;
}
- doLog("<ctr>", null, "INIT: meta_timeout", meta_timeout,
"broker-jmx-port", broker_jmx_port);
-
this.monitor = new UimaAsServiceMonitor(endpoint, broker_host,
broker_jmx_port);
}
@@ -146,8 +126,8 @@ public class UimaAsPing
}
private void doLog(String methodName, Object ... msg)
- {
- if ( ! enable_log ) return;
+ {
+ if ( !log_enabled ) return;
StringBuffer buf = new StringBuffer(methodName);
for ( Object o : msg ) {
@@ -161,17 +141,6 @@ public class UimaAsPing
System.out.println(buf);
}
- private String fmtArray(int[] array)
- {
- Object[] vals = new Object[array.length];
- StringBuffer sb = new StringBuffer();
-
- for ( int i = 0; i < array.length; i++ ) {
- sb.append("%3s ");
- vals[i] = Integer.toString(array[i]);
- }
- return String.format(sb.toString(), vals);
- }
void evaluateService(IServiceStatistics stats)
{
@@ -182,55 +151,12 @@ public class UimaAsPing
monitor.collect();
stats.setHealthy(true); // this pinger defines 'healthy' as
// 'service responds to get-meta and
broker returns jmx stats'
-
-
- monitor_rate =
Integer.parseInt(smState.getProperty("monitor-rate") ) / 60000; //
convert to minutes
- service_id = Long.parseLong(smState.getProperty("service-id"));
- if (monitor_rate <= 0 ) monitor_rate = 1;
// minimum 1 minute allowed
-
- // Calculate total instance failures within some configured
window. If we get a cluster
- // of failures, signal excessive failures so SM stops spawning new
ones.
- int failures =
Integer.parseInt(smState.getProperty("run-failures"));
- doLog(methodName, "run-failures:", failures);
- if ( (failure_window != null) && (failures > 0) ) {
- int diff = failures - current_failures; // nfailures since
last update
- current_failures = failures;
-
- if ( diff > 0 ) {
- failure_window[failure_cursor++] = diff;
- } else {
- failure_window[failure_cursor++] = 0;
- }
-
- doLog(methodName, "failures", failures, "current_failures",
current_failures,
- "failure_window", fmtArray(failure_window),
"failure_cursor", failure_cursor);
-
- failure_cursor = failure_cursor % failure_window_size;
-
-
-
- int windowed_failures = 0;
- excessive_failures = false;
- for ( int i = 0; i < failure_window_size; i++ ) {
- windowed_failures += failure_window[i];
- }
- if ( windowed_failures >= failure_max ) {
- excessive_failures = true;
- }
- doLog(methodName, "windowed_failures", windowed_failures,
"excessive_failures", excessive_failures);
- }
-
} catch ( Throwable t ) {
stats.setHealthy(false);
monitor.setJmxFailure(t.getMessage());
}
}
- public boolean isExcessiveFailures()
- {
- return excessive_failures;
- }
-
public IServiceStatistics getStatistics()
{
String methodName = "getStatistics";
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java?rev=1565655&r1=1565654&r2=1565655&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
Fri Feb 7 13:48:17 2014
@@ -145,6 +145,7 @@ class PingDriver
this.log_directory = resolveStringProperty ("log_directory",
ping_props, job_props, null); // cli always puts this int job props, no
default
+
jvm_args_str = jvm_args_str.trim();
if ( jvm_args_str.equals("") ) {
jvm_args = null;
@@ -357,12 +358,30 @@ class PingDriver
}
}
+ void setCommonInitProperties(Properties props)
+ {
+ props.setProperty("monitor-rate" , "" + meta_ping_rate);
+ props.setProperty("service-id" , "" + sset.getId().getFriendly());
+ props.setProperty("failure-max" , "" +
ServiceManagerComponent.failure_max);
+ props.setProperty("failure-window" , "" +
ServiceManagerComponent.failure_window);
+ props.setProperty("do-log" , "" + do_log);
+ }
+
+ void setCommonProperties(Properties props)
+ {
+ props.setProperty("total-instances" , "" + sset.countImplementors());
+ props.setProperty("active-instances", "" + sset.getActiveInstances());
+ props.setProperty("references" , "" + sset.countReferences());
+ props.setProperty("run-failures" , "" + sset.getRunFailures());
+ }
+
void runAsThread()
{
long tid = Thread.currentThread().getId();
String methodName = "runAsThread[" + tid + "]";
AServicePing pinger = null;
+ Properties initProps = new Properties();
Properties props = new Properties();
try {
@@ -381,18 +400,14 @@ class PingDriver
return;
}
- try {
- props.setProperty("total-instances", "" +
sset.countImplementors());
- props.setProperty("active-instances", "" +
sset.getActiveInstances());
- props.setProperty("references", "" + sset.countReferences());
- props.setProperty("run-failures", "" + sset.getRunFailures());
- props.setProperty("monitor-rate", "" + meta_ping_rate);
- props.setProperty("service-id", "" + sset.getId().getFriendly());
-
- pinger.setSmState(props);
- pinger.init(ping_arguments, endpoint);
- while ( ! shutdown ) {
-
+ try {
+ setCommonInitProperties(initProps);
+ pinger.setLogger(logger);
+ pinger.init(ping_arguments, endpoint, initProps);
+
+ while ( ! shutdown ) {
+ setCommonProperties(props);
+ pinger.setSmState(props);
Pong pr = new Pong();
pr.setStatistics(pinger.getStatistics());
@@ -647,13 +662,7 @@ class PingDriver
// Ask for the ping
try {
logger.info(methodName, sset.getId(),
"ExtrnPingDriver: ping OUT");
- props.setProperty("total-instances" , "" +
sset.countImplementors());
- props.setProperty("active-instances", "" +
sset.getActiveInstances());
- props.setProperty("references" , "" +
sset.countReferences());
- props.setProperty("run-failures" , "" +
sset.getRunFailures());
- props.setProperty("monitor-rate" , "" +
meta_ping_rate);
- props.setProperty("service-id" , "" +
sset.getId().getFriendly());
-
+ setCommonProperties(props);
oos.writeObject(new Ping(false, props));
oos.flush();
oos.reset();
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1565655&r1=1565654&r2=1565655&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
Fri Feb 7 13:48:17 2014
@@ -688,8 +688,9 @@ public class ServiceHandler
wanted = instances;
}
+ sset.setNInstances(running + instances);
+ sset.resetRuntimeErrors();
if ( update ) {
- sset.setNInstances(running + instances);
sset.saveMetaProperties();
}
@@ -743,8 +744,8 @@ public class ServiceHandler
tolose = Math.min(instances, running);
}
+ sset.setNInstances(Math.max(0, running - instances)); // never persist
< 0 registered instance
if ( update ) {
- sset.setNInstances(Math.max(0, running - instances)); // never
persist < 0 registered instance
sset.saveMetaProperties();
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1565655&r1=1565654&r2=1565655&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
Fri Feb 7 13:48:17 2014
@@ -98,6 +98,7 @@ public class ServiceManagerComponent
static String default_ping_class;
static int failure_max = 5;
+ static int failure_window = 30;
private String state_dir = null;
private String state_file = null;
@@ -253,6 +254,7 @@ public class ServiceManagerComponent
DuccDaemonRuntimeProperties.getInstance().boot(DaemonName.ServiceManager,getProcessJmxUrl());
failure_max =
SystemPropertyResolver.getIntProperty("ducc.sm.instance.failure.max",
failure_max);
+ failure_window =
SystemPropertyResolver.getIntProperty("ducc.sm.instance.failure.window",
failure_window);
meta_ping_rate =
SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.rate", meta_ping_rate);
meta_ping_timeout =
SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.timeout",
meta_ping_timeout);
meta_ping_stability =
SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.stability",
meta_ping_stability);
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1565655&r1=1565654&r2=1565655&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Fri Feb 7 13:48:17 2014
@@ -470,6 +470,12 @@ public class ServiceSet
return ping_only;
}
+ synchronized void resetRuntimeErrors()
+ {
+ run_failures = 0;
+ excessiveRunFailures = false;
+ }
+
synchronized void setAutostart(boolean auto)
{
cancelLinger();
