Author: challngr
Date: Thu May 8 19:06:20 2014
New Revision: 1593381
URL: http://svn.apache.org/r1593381
Log:
UIMA-3814 First pass, deal with hanging nodes in check_ducc.
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1593381&r1=1593380&r2=1593381&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Thu May 8 19:06:20
2014
@@ -23,6 +23,7 @@ import os
import sys
from time import time
import getopt
+import signal
from ducc_util import DuccUtil
from ducc_base import DuccProperties
@@ -34,6 +35,11 @@ from ducc_util import ThreadPool
class CheckDucc(DuccUtil):
+
+ def __init__(self):
+ DuccUtil.__init__(self)
+ self.badnodes = []
+
def validate(self, checkdate):
verify_slave_node(checkdate, self.ducc_properties)
self.check_clock_skew(checkdate)
@@ -97,20 +103,34 @@ class CheckDucc(DuccUtil):
messages.append((spacer, 'no processes found.'))
if ( self.kill_signal == None ):
+ response = "Node health checks return."
if ( self.single_user ) :
lines = self.ssh(node, True, self.DUCC_HOME +
"/admin/check_ducc", "-s", "-x", str(int(time())))
else:
lines = self.ssh(node, True, self.DUCC_HOME +
"/admin/check_ducc", "-x", str(int(time())))
while 1:
line = lines.readline()
+ if ( 'signal' in line ):
+ response = "Node health did not complete: " + line
+ self.badnodes.append(node)
if ( not line ):
break
line = line.strip()
messages.append((spacer, line))
#messages.append((spacer, '[]', line))
+ messages.append((spacer, response))
return messages
+ def signalHandler(self, signum, frame):
+ print "-------- Caught signal", signum, "--------"
+ if ( len(self.badnodes) != 0 ):
+ print "Health checks on these nodes did not return:"
+ for n in self.badnodes:
+ print n,
+ print ''
+ sys.exit(1)
+
def usage(self, msg):
if ( msg != None ):
print msg
@@ -297,6 +317,8 @@ class CheckDucc(DuccUtil):
self.threadpool = ThreadPool(n_nodes + 5) # more for the head
processes
checked = {}
+ signal.signal(signal.SIGINT, self.signalHandler)
+
try:
for (nodefile, nodelist) in nodes.items():
if ( nodelist == None ):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1593381&r1=1593380&r2=1593381&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Thu May 8
19:06:20 2014
@@ -411,37 +411,38 @@ class DuccUtil(DuccBase):
def ssh_ok(self, node, line):
spacer = ' '
+ messages = []
if ( line.startswith("Permission denied") ):
- print ' '
- print spacer, "ALERT: Passwordless SSH is not configured correctly
for node", node
- print spacer, "ALERT: SSH returns '" + line + "'"
- return False
+ messages.append(' ')
+ messages.append(spacer + "ALERT: Passwordless SSH is not
configured correctly for node " + node)
+ messages.append(spacer + "ALERT: SSH returns '" + line + "'")
+ return messages
if ( line.startswith("Host key verification failed") ):
- print ' '
- print spacer, "ALERT: Passwordless SSH is not configured correctly
for node", node
- print spacer, "ALERT: SSH returns '" + line + "'"
- return False
+ messages.append(' ')
+ messages.append(spacer + "ALERT: Passwordless SSH is not
configured correctly for node " + node)
+ messages.append(spacer + "ALERT: SSH returns '" + line + "'")
+ return messages
if ( line.find("Connection refused") >= 0 ):
- print ' '
- print spacer, "ALERT: SSH is not not enabled on node", node
- print spacer, "ALERT: SSH returns '" + line + "'"
- return False
+ messages.append(' ')
+ messages.append(spacer + "ALERT: SSH is not not enabled on node "
+ node)
+ messages.append(spacer + "ALERT: SSH returns '" + line + "'")
+ return messages
if ( line.find("Connection timed") >= 0 ):
- print ' '
- print spacer, "\nALERT: SSH did not respond with timeout of 10
secnds", node
- print spacer, "ALERT: SSH returns '" + line + "'"
- return False
+ messages.append(' ')
+ messages.append(spacer + "\nALERT: SSH did not respond with
timeout of 10 secnds " + node)
+ messages.append(spacer + "ALERT: SSH returns '" + line + "'")
+ return messages
if ( line.find("No route") >= 0 ):
- print ' '
- print spacer, 'ALERT: SSH cannot connect to host.'
- print spacer, "ALERT: SSH returns '" + line + "'"
- return False
+ messages.append(' ')
+ messages.append(spacer + 'ALERT: SSH cannot connect to host.')
+ messages.append(spacer + "ALERT: SSH returns '" + line + "'")
+ return messages
- return True
+ return None
#
# Input is array lines from ps command looking for ducc processes owned
this user.
@@ -466,7 +467,10 @@ class DuccUtil(DuccBase):
if ( line.startswith('PID')):
continue
- if ( not self.ssh_ok(line, node) ):
+ ssh_errors = self.ssh_ok(line, node)
+ if ( ssh_errors != None ):
+ for m in ssh_errors:
+ print m
ok = False
continue
