Author: challngr
Date: Tue Oct 1 17:14:40 2013
New Revision: 1528151
URL: http://svn.apache.org/r1528151
Log:
UIMA-3301 Add --nothreading option and automatically disable threading if
python level < 2.6
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1528151&r1=1528150&r2=1528151&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue Oct 1 17:14:40
2013
@@ -146,6 +146,9 @@ class CheckDucc(DuccUtil):
print " -s localdate"
print " Validate the local installation, called via ssh usually.
The date is the dat on the calling machine."
print ""
+ print " --nothreading"
+ print " Disable multithreaded operation if it would otherwise
be used"
+ print ""
print " -v --verbose"
print " If specified, print the validated configuration to the
console."
print ""
@@ -155,7 +158,7 @@ class CheckDucc(DuccUtil):
def main(self, argv):
try:
- opts, args = getopt.getopt(argv, 'cikn:opqrs:u:h?v',
['--configuration', '--nodelist=', '--user=', '--int', '--quit', '--kill',
'--pids', '--reap', '--verbose' ])
+ opts, args = getopt.getopt(argv, 'cikn:opqs:h?v',
['configuration', 'nodelist=', 'int', 'quit', 'kill', 'pids', 'verbose',
'nothreading' ])
except:
self.usage("Invalid arguments " + ' '.join(argv))
@@ -189,6 +192,8 @@ class CheckDucc(DuccUtil):
print 'Conflicting kill signals: -KILL and',
self.kill_signal
return
self.kill_signal = '-KILL'
+ elif o in ( '--nothreading' ):
+ self.disable_threading()
elif o in ('-p', '--pids'):
redo_pids = True
elif o in ('-s'):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1528151&r1=1528150&r2=1528151&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue Oct 1 17:14:40 2013
@@ -200,7 +200,7 @@ class Ducc(DuccUtil):
pid = self.nohup(cmd)
else:
pid = self.spawn(' '.join(cmd))
- print 'PID ' + pid # nohup will print this from the
(twice) forked process if background
+ print 'PID ' + str(pid) # nohup will print this from the
(twice) forked process if background
# hard for us to access it here in
nohup
if ( (c == 'ws') or ( c == 'viz') ):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1528151&r1=1528150&r2=1528151&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Oct 1
17:14:40 2013
@@ -27,6 +27,7 @@ import re
import grp
import resource
import time
+import platform
from threading import *
import traceback
@@ -51,6 +52,9 @@ sys.path.append(DUCC_HOME + '/bin')
from ducc_base import DuccBase
from ducc_base import DuccProperties
+global use_threading
+use_threading = True
+
class ThreadWorker(Thread):
def __init__(self, queue, outlock):
Thread.__init__(self)
@@ -80,29 +84,38 @@ class ThreadWorker(Thread):
class ThreadPool:
def __init__(self, size):
- self.size = size
- self.queue = Queue.Queue()
- outlock = Lock()
-
- MAX_NPSIZE = 100
- if ( self.size > MAX_NPSIZE ):
- self.size = MAX_NPSIZE
-
- for i in range(self.size):
- worker = ThreadWorker(self.queue, outlock)
- worker.start()
+ if ( use_threading ):
+ self.size = size
+ self.queue = Queue.Queue()
+ outlock = Lock()
+
+ MAX_NPSIZE = 100
+ if ( self.size > MAX_NPSIZE ):
+ self.size = MAX_NPSIZE
+
+ for i in range(self.size):
+ worker = ThreadWorker(self.queue, outlock)
+ worker.start()
def invoke(self, method, *args):
- self.queue.put((method, args))
- pass
+ if ( use_threading ):
+ self.queue.put((method, args))
+ else:
+ response = method(args)
+ if ( response != None and len(response) > 0):
+ for l in response:
+ print ' '.join(l)
def quit(self):
- for i in range(self.size):
- self.queue.put((None, 'quit'))
-
- print "Waiting for Completion"
- self.queue.join()
- print "All threads returned"
+ if ( use_threading ):
+ for i in range(self.size):
+ self.queue.put((None, 'quit'))
+
+ print "Waiting for Completion"
+ self.queue.join()
+ print "All threads returned"
+ else:
+ print 'All Work completed'
class DuccUtil(DuccBase):
@@ -579,7 +592,7 @@ class DuccUtil(DuccBase):
return lines.readline().strip()
def show_ducc_environment(self):
-
+ global use_threading
#
# Print the java version
#
@@ -601,6 +614,7 @@ class DuccUtil(DuccBase):
response.append('ENV: ' + line.strip())
+ response.append('ENV: Threading enabled: ' + str(use_threading))
#
# Get the total memory for the node
#
@@ -726,7 +740,12 @@ class DuccUtil(DuccBase):
return (rc == 0)
+ def disable_threading(self):
+ global use_threading
+ use_threading = False
+
def __init__(self):
+ global use_threading
DuccBase.__init__(self)
self.duccling = None
self.broker_url = 'tcp://localhost:61616'
@@ -748,6 +767,16 @@ class DuccUtil(DuccBase):
if (manage_broker in ('t', 'true', 'T', 'True')) :
self.automanage = True
+ py_version = platform.python_version().split('.')
+ if ( int(py_version[0]) > 2 ):
+ print "Warning, only Python Version 2 is supported."
+ if ( int(py_version[1]) < 4 ):
+ print "Python must be at least at version 2.4."
+ sys.exit(1)
+ if ( int(py_version[1]) < 6 ):
+ use_threading = False
+
+
if __name__ == "__main__":
util = DuccUtil()
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1528151&r1=1528150&r2=1528151&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue Oct 1 17:14:40
2013
@@ -99,7 +99,7 @@ class StartDucc(DuccUtil):
line = lines.readline().strip()
if ( not line ):
break
- msgs.append(('[]', line))
+ #msgs.append(('[]', line))
if ( line.startswith('PID') ):
toks = line.split(' ') # get the PID
msgs.append((' PID', toks[1]))
@@ -179,9 +179,6 @@ class StartDucc(DuccUtil):
print ""
print " start_ducc -n foo.nodes -n bar.nodes -n baz.nodes"
print ""
- print " -m --management"
- print " Start the management processes (rm, sm, pm, webserver,
orchestrator) on the local node."
- print ""
print " -s --singleuser"
print " Start ducc in 'single user mode'. This bypasses some
checking required for multi-user"
print " mode and not required for single-user mode."
@@ -203,6 +200,9 @@ class StartDucc(DuccUtil):
print " viz - visualization server"
print " agent - node agent"
print ""
+ print " --nothreading"
+ print " Disable multithreaded operation if it would otherwise
be used"
+ print ""
print "Examples:"
print " Start all DUCC processes, using custom nodelists:"
print " start_ducc -m -n foo.nodes -n bar.nodes"
@@ -229,10 +229,6 @@ class StartDucc(DuccUtil):
def main(self, argv):
- environ = self.show_ducc_environment()
- for e in environ:
- print e
-
if ( not self.verify_jvm() ):
sys.exit(1);
@@ -240,26 +236,25 @@ class StartDucc(DuccUtil):
nodefiles = []
components = []
- management = False
single_user = False
or_parms = self.ducc_properties.get('ducc.orchestrator.start.type')
self.pids = DuccProperties()
self.pids.load_if_exists(self.pid_file)
try:
- opts, args = getopt.getopt(argv, 'c:mn:sh?v', ['component=',
'components=', 'help', 'nodelist=', 'management', 'singleuser', 'cold', 'warm',
'hot'])
+ opts, args = getopt.getopt(argv, 'c:mn:sh?v', ['component=',
'components=', 'help', 'nodelist=', 'singleuser', 'cold', 'warm', 'hot',
'nothreading'])
except:
self.invalid('Invalid arguments', ' '.join(argv))
for ( o, a ) in opts:
if o in ( '-c', '--components' ):
components.append(a)
- elif o in ( '-m', '--management' ):
- management = True
elif o in ( '-n', '--nodelist' ):
nodefiles.append(a)
elif o in ( '-s', '--singleuser' ):
single_user = True
+ elif o in ( '--nothreading' ):
+ self.disable_threading()
elif o in ( '--cold', '--warm', '--hot' ):
or_parms = o[2:] # (strip the leading --)
elif ( o == '-v'):
@@ -272,22 +267,14 @@ class StartDucc(DuccUtil):
else:
self.invalid('bad args: ', ' '.join(argv))
- # 'management' means start all the management daemons - if specific
components are also specified
- # there is at least a redundancy and maybe also a conflict.
- if ( (len(components) != 0) and management ):
- self.invalid("The [-m | --management] and [-c | --component]
options are mutually exclusive.")
+ environ = self.show_ducc_environment()
+ for e in environ:
+ print e
# no args, or just -s - make equivalent of -management and
-nodefile=DUCC.HOME/resources/ducc.nodes
- if ( (len(argv) == 0) or ((len(argv) == 1) and single_user) ):
+ if ( (len(components) == 0) and (len(nodefiles) == 0 ) ) :
nodefiles = self.default_nodefiles
components = self.default_components
- must_verify_nodepools = True
- else:
- must_verify_nodepools = False
-
- # this means all the non-agent processes - conflicts are already
checked
- if ( management ):
- components = self.default_components
self.verify_required_directories()
@@ -298,9 +285,6 @@ class StartDucc(DuccUtil):
print 'FAIL: Cannot run javac to run java verification'
return
- #print 'nodefiles:', nodefiles
- #print 'components:', components
-
# make sure all the nodefiles exist and are readable
ok = True
nodes = {}
@@ -359,7 +343,6 @@ class StartDucc(DuccUtil):
if ( len(components) != 0 ):
print 'Starting', or_parms
-
for com in components:
if ( com == 'broker' ):
pass # already started
@@ -372,8 +355,6 @@ class StartDucc(DuccUtil):
traceback.print_exc()
print "Errors encounted, DUCC may not be started
correctly."
sys.exit(1)
- else:
- print 'Not starting management components.'
self.threadpool.quit()
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1528151&r1=1528150&r2=1528151&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Tue Oct 1 17:14:40
2013
@@ -146,6 +146,11 @@ class StopDucc(DuccUtil):
print ' -k --kill'
print ' Stop the component forcibly and immediately using kill
-9. Use this only if a'
print ' normal stop does not work (e.g. the process may be
hung).'
+ print ''
+ print ' --nothreading'
+ print ' Disable multithreaded operation if it would otherwise
be used'
+ print ''
+
sys.exit(1)
def invalid(self, *msg):
@@ -174,7 +179,7 @@ class StopDucc(DuccUtil):
all = False
try:
- opts, args = getopt.getopt(argv, 'ac:n:kmmn:qh?v', ['all',
'component=', 'components=', 'help', 'nodelist=', 'management', 'kill',
'quiesce'])
+ opts, args = getopt.getopt(argv, 'ac:n:kmmn:qh?v', ['all',
'component=', 'components=', 'help', 'nodelist=', 'management', 'kill',
'quiesce', 'nothreading'])
except:
self.invalid('Invalid arguments ' + ' '.join(argv))
@@ -195,6 +200,8 @@ class StopDucc(DuccUtil):
force = True
elif o in ( '-q', '--quiesce' ):
quiesce = True
+ elif o in ( '--nothreading' ):
+ self.disable_threading()
elif ( o == '-v' ) :
print self.version()
sys.exit(0)
