Author: challngr Date: Tue Oct 1 17:14:40 2013 New Revision: 1528151 URL: http://svn.apache.org/r1528151 Log: UIMA-3301 Add --nothreading option and automatically disable threading if python level < 2.6
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1528151&r1=1528150&r2=1528151&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue Oct 1 17:14:40 2013 @@ -146,6 +146,9 @@ class CheckDucc(DuccUtil): print " -s localdate" print " Validate the local installation, called via ssh usually. The date is the dat on the calling machine." print "" + print " --nothreading" + print " Disable multithreaded operation if it would otherwise be used" + print "" print " -v --verbose" print " If specified, print the validated configuration to the console." print "" @@ -155,7 +158,7 @@ class CheckDucc(DuccUtil): def main(self, argv): try: - opts, args = getopt.getopt(argv, 'cikn:opqrs:u:h?v', ['--configuration', '--nodelist=', '--user=', '--int', '--quit', '--kill', '--pids', '--reap', '--verbose' ]) + opts, args = getopt.getopt(argv, 'cikn:opqs:h?v', ['configuration', 'nodelist=', 'int', 'quit', 'kill', 'pids', 'verbose', 'nothreading' ]) except: self.usage("Invalid arguments " + ' '.join(argv)) @@ -189,6 +192,8 @@ class CheckDucc(DuccUtil): print 'Conflicting kill signals: -KILL and', self.kill_signal return self.kill_signal = '-KILL' + elif o in ( '--nothreading' ): + self.disable_threading() elif o in ('-p', '--pids'): redo_pids = True elif o in ('-s'): Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1528151&r1=1528150&r2=1528151&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue Oct 1 17:14:40 2013 @@ -200,7 +200,7 @@ class Ducc(DuccUtil): pid = self.nohup(cmd) else: pid = self.spawn(' '.join(cmd)) - print 'PID ' + pid # nohup will print this from the (twice) forked process if background + print 'PID ' + str(pid) # nohup will print this from the (twice) forked process if background # hard for us to access it here in nohup if ( (c == 'ws') or ( c == 'viz') ): Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1528151&r1=1528150&r2=1528151&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Oct 1 17:14:40 2013 @@ -27,6 +27,7 @@ import re import grp import resource import time +import platform from threading import * import traceback @@ -51,6 +52,9 @@ sys.path.append(DUCC_HOME + '/bin') from ducc_base import DuccBase from ducc_base import DuccProperties +global use_threading +use_threading = True + class ThreadWorker(Thread): def __init__(self, queue, outlock): Thread.__init__(self) @@ -80,29 +84,38 @@ class ThreadWorker(Thread): class ThreadPool: def __init__(self, size): - self.size = size - self.queue = Queue.Queue() - outlock = Lock() - - MAX_NPSIZE = 100 - if ( self.size > MAX_NPSIZE ): - self.size = MAX_NPSIZE - - for i in range(self.size): - worker = ThreadWorker(self.queue, outlock) - worker.start() + if ( use_threading ): + self.size = size + self.queue = Queue.Queue() + outlock = Lock() + + MAX_NPSIZE = 100 + if ( self.size > MAX_NPSIZE ): + self.size = MAX_NPSIZE + + for i in range(self.size): + worker = ThreadWorker(self.queue, outlock) + worker.start() def invoke(self, method, *args): - self.queue.put((method, args)) - pass + if ( use_threading ): + self.queue.put((method, args)) + else: + response = method(args) + if ( response != None and len(response) > 0): + for l in response: + print ' '.join(l) def quit(self): - for i in range(self.size): - self.queue.put((None, 'quit')) - - print "Waiting for Completion" - self.queue.join() - print "All threads returned" + if ( use_threading ): + for i in range(self.size): + self.queue.put((None, 'quit')) + + print "Waiting for Completion" + self.queue.join() + print "All threads returned" + else: + print 'All Work completed' class DuccUtil(DuccBase): @@ -579,7 +592,7 @@ class DuccUtil(DuccBase): return lines.readline().strip() def show_ducc_environment(self): - + global use_threading # # Print the java version # @@ -601,6 +614,7 @@ class DuccUtil(DuccBase): response.append('ENV: ' + line.strip()) + response.append('ENV: Threading enabled: ' + str(use_threading)) # # Get the total memory for the node # @@ -726,7 +740,12 @@ class DuccUtil(DuccBase): return (rc == 0) + def disable_threading(self): + global use_threading + use_threading = False + def __init__(self): + global use_threading DuccBase.__init__(self) self.duccling = None self.broker_url = 'tcp://localhost:61616' @@ -748,6 +767,16 @@ class DuccUtil(DuccBase): if (manage_broker in ('t', 'true', 'T', 'True')) : self.automanage = True + py_version = platform.python_version().split('.') + if ( int(py_version[0]) > 2 ): + print "Warning, only Python Version 2 is supported." + if ( int(py_version[1]) < 4 ): + print "Python must be at least at version 2.4." + sys.exit(1) + if ( int(py_version[1]) < 6 ): + use_threading = False + + if __name__ == "__main__": util = DuccUtil() Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1528151&r1=1528150&r2=1528151&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue Oct 1 17:14:40 2013 @@ -99,7 +99,7 @@ class StartDucc(DuccUtil): line = lines.readline().strip() if ( not line ): break - msgs.append(('[]', line)) + #msgs.append(('[]', line)) if ( line.startswith('PID') ): toks = line.split(' ') # get the PID msgs.append((' PID', toks[1])) @@ -179,9 +179,6 @@ class StartDucc(DuccUtil): print "" print " start_ducc -n foo.nodes -n bar.nodes -n baz.nodes" print "" - print " -m --management" - print " Start the management processes (rm, sm, pm, webserver, orchestrator) on the local node." - print "" print " -s --singleuser" print " Start ducc in 'single user mode'. This bypasses some checking required for multi-user" print " mode and not required for single-user mode." @@ -203,6 +200,9 @@ class StartDucc(DuccUtil): print " viz - visualization server" print " agent - node agent" print "" + print " --nothreading" + print " Disable multithreaded operation if it would otherwise be used" + print "" print "Examples:" print " Start all DUCC processes, using custom nodelists:" print " start_ducc -m -n foo.nodes -n bar.nodes" @@ -229,10 +229,6 @@ class StartDucc(DuccUtil): def main(self, argv): - environ = self.show_ducc_environment() - for e in environ: - print e - if ( not self.verify_jvm() ): sys.exit(1); @@ -240,26 +236,25 @@ class StartDucc(DuccUtil): nodefiles = [] components = [] - management = False single_user = False or_parms = self.ducc_properties.get('ducc.orchestrator.start.type') self.pids = DuccProperties() self.pids.load_if_exists(self.pid_file) try: - opts, args = getopt.getopt(argv, 'c:mn:sh?v', ['component=', 'components=', 'help', 'nodelist=', 'management', 'singleuser', 'cold', 'warm', 'hot']) + opts, args = getopt.getopt(argv, 'c:mn:sh?v', ['component=', 'components=', 'help', 'nodelist=', 'singleuser', 'cold', 'warm', 'hot', 'nothreading']) except: self.invalid('Invalid arguments', ' '.join(argv)) for ( o, a ) in opts: if o in ( '-c', '--components' ): components.append(a) - elif o in ( '-m', '--management' ): - management = True elif o in ( '-n', '--nodelist' ): nodefiles.append(a) elif o in ( '-s', '--singleuser' ): single_user = True + elif o in ( '--nothreading' ): + self.disable_threading() elif o in ( '--cold', '--warm', '--hot' ): or_parms = o[2:] # (strip the leading --) elif ( o == '-v'): @@ -272,22 +267,14 @@ class StartDucc(DuccUtil): else: self.invalid('bad args: ', ' '.join(argv)) - # 'management' means start all the management daemons - if specific components are also specified - # there is at least a redundancy and maybe also a conflict. - if ( (len(components) != 0) and management ): - self.invalid("The [-m | --management] and [-c | --component] options are mutually exclusive.") + environ = self.show_ducc_environment() + for e in environ: + print e # no args, or just -s - make equivalent of -management and -nodefile=DUCC.HOME/resources/ducc.nodes - if ( (len(argv) == 0) or ((len(argv) == 1) and single_user) ): + if ( (len(components) == 0) and (len(nodefiles) == 0 ) ) : nodefiles = self.default_nodefiles components = self.default_components - must_verify_nodepools = True - else: - must_verify_nodepools = False - - # this means all the non-agent processes - conflicts are already checked - if ( management ): - components = self.default_components self.verify_required_directories() @@ -298,9 +285,6 @@ class StartDucc(DuccUtil): print 'FAIL: Cannot run javac to run java verification' return - #print 'nodefiles:', nodefiles - #print 'components:', components - # make sure all the nodefiles exist and are readable ok = True nodes = {} @@ -359,7 +343,6 @@ class StartDucc(DuccUtil): if ( len(components) != 0 ): print 'Starting', or_parms - for com in components: if ( com == 'broker' ): pass # already started @@ -372,8 +355,6 @@ class StartDucc(DuccUtil): traceback.print_exc() print "Errors encounted, DUCC may not be started correctly." sys.exit(1) - else: - print 'Not starting management components.' self.threadpool.quit() Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1528151&r1=1528150&r2=1528151&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Tue Oct 1 17:14:40 2013 @@ -146,6 +146,11 @@ class StopDucc(DuccUtil): print ' -k --kill' print ' Stop the component forcibly and immediately using kill -9. Use this only if a' print ' normal stop does not work (e.g. the process may be hung).' + print '' + print ' --nothreading' + print ' Disable multithreaded operation if it would otherwise be used' + print '' + sys.exit(1) def invalid(self, *msg): @@ -174,7 +179,7 @@ class StopDucc(DuccUtil): all = False try: - opts, args = getopt.getopt(argv, 'ac:n:kmmn:qh?v', ['all', 'component=', 'components=', 'help', 'nodelist=', 'management', 'kill', 'quiesce']) + opts, args = getopt.getopt(argv, 'ac:n:kmmn:qh?v', ['all', 'component=', 'components=', 'help', 'nodelist=', 'management', 'kill', 'quiesce', 'nothreading']) except: self.invalid('Invalid arguments ' + ' '.join(argv)) @@ -195,6 +200,8 @@ class StopDucc(DuccUtil): force = True elif o in ( '-q', '--quiesce' ): quiesce = True + elif o in ( '--nothreading' ): + self.disable_threading() elif ( o == '-v' ) : print self.version() sys.exit(0)