Author: challngr Date: Tue Jan 22 16:22:39 2013 New Revision: 1437025 URL: http://svn.apache.org/viewvc?rev=1437025&view=rev Log: UIMA-2584 Scripting to do post-installation tasks as described.
Updates to all scripting components to auto-manage the ActiveMQ server using common configuration in ducc.properties. Updates to default ducc.properties for the above. Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (with props) Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue Jan 22 16:22:39 2013 @@ -40,6 +40,12 @@ class CheckDucc(DuccUtil): def validate(self, checkdate): verify_slave_node(checkdate, self.ducc_properties) + def verify_activemq(self): + if ( self.is_amq_active() ): + print 'ActiveMQ is found listening at', self.broker_protocol + "://" + self.broker_host + ':' + self.broker_port + return True + return False + def usage(self, msg): if ( msg != None ): print msg @@ -122,7 +128,6 @@ class CheckDucc(DuccUtil): print 'badarg', a usage('bad arg: ' + a) - if ( do_validate ): # if validating, ONLY validate, called via ssh usually self.validate(checkdate) @@ -131,15 +136,17 @@ class CheckDucc(DuccUtil): if ( reap and (user == 'ducc') ): usage('Can only reap non-udcc users') - jvm = self.ducc_properties.get('ducc.jvm') if ( jvm == None ): - print 'WARN: "ducc.jvm is not specified in ducc.properties. Default is simply "java" which may not work on all nodes.' + print 'WARN: ducc.jvm is not specified in ducc.properties. Default is simply "java" which may not work on all nodes.' if ( not verify_master_node(self.ducc_properties) ): - print 'FAIL: Cannot run javac to build the java test' + print 'FAIL: Cannot verify master mode' return - + + if ( not self.verify_activemq() ): + print 'ActiveMQ is not running on', self.broker_protocol + "://" + self.broker_host + ':' + self.broker_port + # init the PID file pids = DuccProperties() pids.load_if_exists(self.pid_file) @@ -238,6 +245,7 @@ class CheckDucc(DuccUtil): return if ( kill ): + self.stop_broker() self.remove_orchestrator_lock() if ( len(pids) == 0): Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue Jan 22 16:22:39 2013 @@ -29,7 +29,29 @@ from ducc_util import DuccProperties from local_hooks import verify_slave_node class Ducc(DuccUtil): + + def run_broker(self, component): + broker_port = self.ducc_properties.get('ducc.broker.port') + broker_url_decoration = self.ducc_properties.get('ducc.broker.server.url.decoration') + broker_memory_opts = self.ducc_properties.get('ducc.broker.memory.options') + broker_config = self.ducc_properties.get('ducc.broker.configuration') + broker_home = self.ducc_properties.get('ducc.broker.home') + + if ( broker_config[0] != '/' ): # relative to broker_home if not absolute + broker_config = broker_home + '/' + broker_config + + os.environ['ACTIVEMQ_OPTS'] = '-DDUCC_AMQ_PORT=' + broker_port + ' -DDUCC_AMQ_DECORATION=' + broker_url_decoration + os.environ['ACTIVEMQ_OPTS_MEMORY'] = broker_memory_opts + os.environ['ACTIVEMQ_HOME'] = broker_home + + here = os.getcwd() + os.chdir(broker_home + '/bin') + CMD = './activemq start xbean:' + broker_config + self.spawn(CMD) + os.chdir(here) + print "Started AMQ broker" + def run_component(self, component, or_parms, numagents, rmoverride, background, nodup, localdate): if ( component == 'all' ): @@ -38,9 +60,14 @@ class Ducc(DuccUtil): complist = component.split(',') args = None + # ducc-head needs to be in system properties before the ducc daemon reads ducc.properties + # to insure it can be substituted properly + ducc_head = self.ducc_properties.get('ducc.head') + jvm_opts = [] jvm_opts.append('-Dos.page.size=' + self.os_pagesize) jvm_opts.append('-Dducc.deploy.configuration=' + self.DUCC_HOME + '/resources/ducc.properties') + jvm_opts.append('-Dducc.head=' + ducc_head) service = 'org.apache.uima.ducc.common.main.DuccService' for c in complist: @@ -163,7 +190,7 @@ class Ducc(DuccUtil): print ' ducc.py -k' print 'Where:' print ' -c <component> is the name of the comp[onent to start, currently one of' - print ' agent rm sm pm ws orchestrator' + print ' agent rm sm pm ws orchestrator broker' print ' -- or --' print ' all - to start rm sm pm ws orchestrator' print ' NOTE -- that agents should be started separately' @@ -172,6 +199,9 @@ class Ducc(DuccUtil): print ' -n <numagents> if > 1, multiple agents are started (testing mode)' print ' -o <mem-in-GB> rm memory override for use on small machines' print ' -k causes the entire DUCC system to shutdown' + print ' --nodup If specified, do not start a process if it appears to be already started.' + print ' --or_parms [cold|warm|hot]' + print ' --ducc_head nodename the name of the "ducc head" where ducc is started from' print ' arguments - any additional arguments to pass to the component.' sys.exit(1) @@ -188,7 +218,7 @@ class Ducc(DuccUtil): localdate = 0 try: - opts, args = getopt.getopt(argv, 'bc:d:n:o:k?v', ['or_parms=', 'nodup']) + opts, args = getopt.getopt(argv, 'bc:d:n:o:k?v', ['or_parms=', 'nodup' ]) except: self.usage('Bad arguments ' + ' '.join(argv)) @@ -196,8 +226,7 @@ class Ducc(DuccUtil): if ( o == '-c' ) : component = a if ( component == 'or' ): - component = 'orchestrator' - + component = 'orchestrator' elif ( o == '-b'): background = True elif ( o == '-d'): @@ -217,18 +246,21 @@ class Ducc(DuccUtil): else: print 'badarg', a usage('bad arg: ' + a) - + if ( shutdown ): if ( component != None ): print 'Note: -c flag for component not allowed when shutting down. Shutdown aborted' sys.exit(1); self.clean_shutdown(); sys.exit(1) - + if ( component == None ): self.usage("Must specify component") - self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate) + if ( component == 'broker' ): + self.run_broker(background) + else: + self.run_component(component, or_parms, numagents, rmoverride, background, nodup, localdate) return def __call__(self, *args): Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1437025&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (added) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Tue Jan 22 16:22:39 2013 @@ -0,0 +1,253 @@ +#!/usr/bin/env python +# ----------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------- + + +import os +import sys +import shutil +import subprocess + +from ducc_boot import * +set_ducc_home() + +from ducc_util import DuccUtil +from ducc_util import DuccProperties +from ducc import Ducc + +class PostInstall(DuccUtil): + + def usage(self, msg): + + if ( msg != None ): + print ' '.join(msg) + + + print "Usage:" + print " stop_sim [options]" + print " If no options are given, all ducc processes from sim.pids are stopped using SIGINT" + print "" + print "Options:" + print " -c --component compenent name" + print " The name is one of", self.default_components, "or an ID from sim.pids or the nodelist" + print " that was used to start DUCC" + print "" + print " -i --instance instancid" + print " Sends SIGINT to the indicated process, enabling orderly shutdown." + print "" + print " -k --kill" + print " Sends SIGKILL to the indicated process" + print "" + print " -p --pause" + print " Sends SIGSTOP to the indicated process" + print "" + print " -r --resume" + print " Sends SIGCONT to the indicated process" + print "" + print " -v, --version" + print " Print the current DUCC version" + print "" + sys.exit(1) + + + def fail(self, *msg): + print ' '.join(msg) + print "POST INSTALLATION FAILED" + sys.exit(1) + + + def warn(self, *msg): + print '' + print 'WARNING' + print 'WARNing', ' '.join(msg) + print 'WARNING' + print '' + + def create_keystore(self, keytool): + + ''' + CN - Common Name of the certificate owner + OU - Organizational Unit of the certificate owner + O - Organization to which the certificate owner belongs + L - Locality name of the certificate owner + S - State or province of the certificate owner + C - Country of the certificate owner + ''' + + keystore = self.DUCC_HOME + "/webserver/etc/keystore" + cmd = 'rm ' + keystore + os.system(cmd); + + pw = self.ducc_properties.get('ducc.ws.port.ssl.pw') + if ( pw == 'quackquack' ): + print self.warn('The default ssl pw of ', pw, ' is configured. For security, change the property \n\tducc.ws.port.ssl.pw to a secure password in ducc.properties.') + + + #/usr/bin/keytool + cmd = self.keytool + cmd += ' ' + cmd += '-genkey' + cmd += ' ' + cmd += '-noprompt' + cmd += ' ' + cmd += '-alias jetty' + cmd += ' ' + cmd += '-dname "CN=org.apache.uima.ducc, OU=uima.ducc, O=Apache, L=Wilmington, S=Delaware, C=USA"' + cmd += ' ' + cmd += '-keyalg RSA' + cmd += ' ' + cmd += '-validity 10000' + cmd += ' ' + cmd += '-keystore ' + keystore + cmd += ' ' + cmd += '-storepass '+pw + cmd += ' ' + cmd += '-keypass '+pw + os.system(cmd); + print 'keystore created.' + + + # Setup and verify amq + # make sure verify_ducc is sufficient - maybe move some checks to there? + + def get_java_bindir(self): + + java = self.java() + if ( java == None ): + self.fail("THe path to java must be configured in ducc.properties as property", 'ducc.jvm') + + if ( not java.startswith('/') ): + self.fail("THe java path in property ducc.jvm must be absolute.") + + # We're going to do more checks here so we don't proceed with bogosities + proc = subprocess.Popen(java + ' -version', shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + lines = [] + for line in proc.stdout: + lines.append(line.strip()) + proc.wait() + rc = proc.returncode + + for line in lines: + print "JAVA: " + line + + vertoks = lines[0].split() + self.java_version = vertoks[-1] + + if ( rc != 0 ): + self.fail("Configured java at '", java, "' is incorrect.") + + return os.path.dirname(java) + + def get_java_version(self): + return self.java_version + + def get_java_keytool(self, bindir): + keytool = bindir + "/keytool" + if ( not os.path.exists(keytool) ): + self.fail("Cannot find keytool in ", bindir + '.', + "Is ducc.jvm configured correctly?") + return keytool + + def check_nodes(self): + nodes = self.DUCC_HOME + "/resources/ducc.nodes" + if ( not os.path.exists(nodes) ): + nf = open(nodes, 'w') + nf.write(self.localhost) + nf.close() + print "Initial", nodes, "created." + else: + print "Verified existence of default nodefile " + nodes + print '' + + jdnodes = self.DUCC_HOME + "/resources/jobdriver.nodes" + if ( not os.path.exists(jdnodes) ): + rn = open(jdnodes, "w") + rn.write(self.localhost) + rn.close() + print "Initial nodepool", jdnodes, "created.\n" + else: + print "Verified existence of default JobDriver nodepool " + jdnodes + print '' + + + def check_activemq(self): + if ( self.automanage ): # the default + print 'ActiveMQ is automanaged on node ', self.broker_url + else: + print 'ActiveMQ is managed by the installation and is not verified.' + + def setup_ducc_head(self): + ducc_head = self.ducc_properties.get('ducc.head') + if ( ducc_head == None ): + ducc_head = self.localhost + self.ducc_properties.put('ducc.head', ducc_head) + propsfile = self.DUCC_HOME + "/resources/ducc.properties" + tmp_propsfile = propsfile + ".tmp" + f = open(propsfile) + tf = open(tmp_propsfile, "w") + tf.write("# ducc.head is the node where the main DUCC daemons run\n") + tf.write("ducc.head = " + self.localhost + "\n\n") + for line in f: + tf.write(line); + f.close() + tf.close() + os.rename(propsfile, propsfile + ".orig") + os.rename(tmp_propsfile, propsfile) + self.read_properties() # force re-eval with ducc.head + + print "Ducc head is configured as", ducc_head, '\n' + + def main(self, args): + + print 'Python version:' + print sys.version + print '' + + py_version = sys.version_info + if ( (py_version[0] != 2) or (py_version[1] < 4) ): + self.fail("Python must be installed at level 2.4 or higher.") + + self.setup_ducc_head() + self.check_nodes() + self.check_activemq() + + # insure java is configured and installed + self.java_bindir = self.get_java_bindir() + self.keytool = self.get_java_keytool(self.java_bindir) + print 'Java version:', self.get_java_version() + if ( not '1.6' in self.get_java_version() ): + print 'WARNING: DUCC is only tested under Java 1.6' + else: + print 'Java is verified.' + print '' + + self.create_keystore(self.keytool) + print 'Web server keystore generated from ducc.properties' + print '' + + # Make duccling + here = os.getcwd() + os.chdir("../duccling/src") + os.system("make clean all") + os.chdir(here) + shutil.copyfile("../duccling/src/ducc_ling", self.DUCC_HOME + "/admin/ducc_ling") + print 'Initial ducc_ling is installed. See the installation guide for multi-user setup.' + +if __name__ == "__main__": + postinstall = PostInstall() + postinstall.main(sys.argv[1:]) Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install ------------------------------------------------------------------------------ svn:executable = * Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Jan 22 16:22:39 2013 @@ -43,22 +43,32 @@ class DuccProperties: self.props = {} # - # Expand ${} values from env or from this properties file itself + # Expand all ${} values from env or from this properties file itself + # The search order is: + # 1 look in this properties file + # 2 look in the environment # def do_subst(self, str): - key = None - p = re.compile("\\$\\{[a-zA-Z0-9_]+\\}") - m = p.match(str) + p = re.compile("\\$\\{[a-zA-Z0-9_\\.\\-]+\\}") + ndx = 0 - if ( m != None ): + response = str + m = p.search(response, ndx) + while ( m != None ): key = m.group()[2:-1] - #print str, m, m.group(), key - val = os.environ[key] - response = string.replace(str, m.group() , val) - else: - response = str + val = None + if ( self.has_key(key) ): + val = self.get(key) + elif ( os.environ.has_key(key) ): + val = os.environ[key] + + if ( val != None ): + response = string.replace(response, m.group() , val) + ndx = m.start()+1 + m = p.search(response, ndx) + return response def mkitem(self, line): @@ -77,7 +87,7 @@ class DuccProperties: key = line[:mobj.start()].strip() val = line[mobj.end():].strip() #print 'NEXT', mobj.start(), 'END', mobj.end(), 'KEY', key, 'VAL', val - val = self.do_subst(val) + # val = self.do_subst(val) # we'll do lazy subst on get instead self.props[key] = val else: self.props[line] = None @@ -121,7 +131,7 @@ class DuccProperties: # def get(self, key): if ( self.props.has_key(key) ): - return self.props[key] + return self.do_subst(self.props[key]) # we'll do lazy subst on get instead return None # @@ -183,12 +193,13 @@ class DuccUtil: self.broker_jmx_port = self.ducc_properties.get('ducc.broker.jmx.port') self.broker_decoration = self.ducc_properties.get('ducc.broker.url.decoration') self.broker_url = self.broker_protocol + '://' + self.broker_host + ':' + self.broker_port - self.agent_jvm_args = self.ducc_properties.get('ducc.agent.jvm.args') - self.ws_jvm_args = self.ducc_properties.get('ducc.ws.jvm.args') - self.pm_jvm_args = self.ducc_properties.get('ducc.pm.jvm.args') - self.rm_jvm_args = self.ducc_properties.get('ducc.rm.jvm.args') - self.sm_jvm_args = self.ducc_properties.get('ducc.sm.jvm.args') - self.or_jvm_args = self.ducc_properties.get('ducc.orchestrator.jvm.args') + self.agent_jvm_args = self.ducc_properties.get('ducc.agent.jvm.args') + self.ws_jvm_args = self.ducc_properties.get('ducc.ws.jvm.args') + self.pm_jvm_args = self.ducc_properties.get('ducc.pm.jvm.args') + self.rm_jvm_args = self.ducc_properties.get('ducc.rm.jvm.args') + self.sm_jvm_args = self.ducc_properties.get('ducc.sm.jvm.args') + self.or_jvm_args = self.ducc_properties.get('ducc.orchestrator.jvm.args') + if ( self.broker_decoration == '' ): self.broker_decoration = None @@ -200,10 +211,7 @@ class DuccUtil: self.webserver_node = self.localhost def java(self): - if ( self.jvm == None ): - return 'java' - else: - return self.jvm + return self.jvm def is_amq_active(self): lines = self.popen('ssh', self.broker_host, 'netstat -an') @@ -220,6 +228,16 @@ class DuccUtil: return True return False + def stop_broker(self): + broker_host = self.ducc_properties.get('ducc.broker.hostname') + broker_home = self.ducc_properties.get('ducc.broker.home') + here = os.getcwd() + CMD = broker_home + '/bin/activemq' + CMD = CMD + ' stop' + print CMD + self.ssh(broker_host, False, CMD) + pass + def version(self): lines = self.popen(self.jvm, ' org.apache.uima.ducc.utils.Version') line = lines.readline().strip() @@ -575,7 +593,8 @@ class DuccUtil: def clean_shutdown(self): DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME + "/resources/ducc.properties " - self.spawn('java', DUCC_JVM_OPTS, 'org.apache.uima.ducc.common.main.DuccAdmin', '--killAll') + DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.head=' + self.ducc_properties.get('ducc.head') + self.spawn(self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.common.main.DuccAdmin', '--killAll') def get_os_pagesize(self): lines = self.popen('/usr/bin/getconf', 'PAGESIZE') @@ -702,15 +721,21 @@ class DuccUtil: self.broker_protocol = 'tcp' self.broker_host = 'localhost' self.broker_port = '61616' - self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'viz'] + self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'viz', 'broker'] self.default_nodefiles = [self.DUCC_HOME + '/resources/ducc.nodes'] self.propsfile = self.DUCC_HOME + '/resources/ducc.properties' self.localhost = os.uname()[1] + os.environ['NodeName'] = self.localhost # to match java code's implicit propery so script and java match self.pid_file = self.DUCC_HOME + '/state/ducc.pids' self.set_classpath() self.read_properties() self.os_pagesize = self.get_os_pagesize() + manage_broker = self.ducc_properties.get('ducc.broker.automanage') + self.automanage = False + if (manage_broker in ('t', 'true', 'T', 'True')) : + self.automanage = True + if __name__ == "__main__": util = DuccUtil() Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue Jan 22 16:22:39 2013 @@ -21,7 +21,7 @@ import os import sys -from time import time +import time from ducc_boot import * set_ducc_home() @@ -35,6 +35,29 @@ from local_hooks import verify_master_no class StartDucc(DuccUtil): + def start_broker(self): + + broker_host = self.ducc_properties.get('ducc.broker.hostname') + print 'broker host', broker_host + lines = self.ssh(broker_host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'broker', "'") + while 1: + line = lines.readline().strip() + if ( not line ): + break + print '[] ' + line + if ( line.startswith('PID') ): + toks = line.split(' ') # get the PID + print "Broker on", broker_host, 'PID', toks[1] + self.pids.put('broker@' + broker_host, toks[1]) + lines.close() + break + + for i in range(0, 9): + if ( self.is_amq_active() ): + return + print 'Waiting for broker', str(i) + time.sleep(1) + def start_component(self, ducc, component, or_parms): node = 'local' @@ -61,7 +84,7 @@ class StartDucc(DuccUtil): if ( node == 'local' ): node = self.localhost - lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-b', or_parms, '-d', str(time()), '--nodup' "'") + lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', com, '-b', or_parms, '-d', str(time.time()), '--nodup', "'") # we'll capture anything that the python shell spews because it may be useful, and then drop the # pipe when we see a PID message while 1: @@ -89,7 +112,7 @@ class StartDucc(DuccUtil): spacer = ' ' print host - lines = self.ssh(host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c' 'agent', '-b', '-d', str(time()), '--nodup', "'") + lines = self.ssh(host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c' 'agent', '-b', '-d', str(time.time()), '--nodup', "'") while 1: line = lines.readline().strip() #print '[]' + line @@ -120,7 +143,6 @@ class StartDucc(DuccUtil): else: print spacer, line - def usage(self, *msg): if ( msg[0] != None ): print ' '.join(msg) @@ -219,7 +241,6 @@ class StartDucc(DuccUtil): else: self.invalid('bad args: ', ' '.join(argv)) - # 'management' means start all the management daemons - if specific components are also specified # there is at least a redundancy and maybe also a conflict. if ( (len(components) != 0) and management ): @@ -257,6 +278,12 @@ class StartDucc(DuccUtil): sys.exit(1) # activeMQ needs to be started externally before starting any DUCC processes + if ( self.automanage and ('broker' in components) ): + if ( self.is_amq_active() ): + print 'ActiveMQ is already running on host and port:', self.broker_host + ':' + self.broker_port, 'NOT restarting' + else: + self.start_broker() + #print 'A--------------------------------------------------------------------------------' if ( self.is_amq_active() ): print 'ActiveMQ is found on configured host and port:', self.broker_host + ':' + self.broker_port @@ -271,7 +298,10 @@ class StartDucc(DuccUtil): if ( len(components) != 0 ): print 'Starting', or_parms for com in components: - self.start_component(ducc, com, or_parms) + if ( com == 'broker' ): + pass # already started + else: + self.start_component(ducc, com, or_parms) else: print 'Not starting management components.' #print 'C--------------------------------------------------------------------------------' Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim Tue Jan 22 16:22:39 2013 @@ -21,6 +21,7 @@ import os import sys +import time import getopt from ducc_boot import * @@ -32,6 +33,29 @@ from ducc import Ducc class StartSim(DuccUtil): + def start_broker(self): + + broker_host = self.ducc_properties.get('ducc.broker.hostname') + print 'broker host', broker_host + lines = self.ssh(broker_host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'broker', "'") + while 1: + line = lines.readline().strip() + if ( not line ): + break + print '[] ' + line + if ( line.startswith('PID') ): + toks = line.split(' ') # get the PID + print "Broker on", broker_host, 'PID', toks[1] + self.pids.put('broker@' + broker_host, toks[1]) + lines.close() + break + + for i in range(0, 9): + if ( self.is_amq_active() ): + return + print 'Waiting for broker', str(i) + time.sleep(1) + def run_local_agent(self, pnode, ip, memory ): memory = int(memory) * 1024 * 1024 # to GB from KB @@ -56,7 +80,12 @@ class StartSim(DuccUtil): # Start admin components rm pm sm ws or, on local node using Ducc.py # def startComponents(self, components, or_parms): + for (com, com) in components.items(): + + if ( com == 'broker' ): + continue + if ( com in ('ws', 'viz') ): node = self.webserver_node else: @@ -279,8 +308,12 @@ class StartSim(DuccUtil): if ( (IP != None) or (memory != None) or ( pseudoname != None )) : self.invalid("Running with a nodelist is not compatible with running a single agent."); + if ( components.get('broker') != None ): + self.start_broker() + if ( len(nodefiles) != 0 ): self.startAgents(nodefiles, instances) + self.startComponents(components, or_parms) self.pids.write('sim.pids') Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Tue Jan 22 16:22:39 2013 @@ -21,6 +21,7 @@ import os import sys +import time from ducc_boot import * set_ducc_home() @@ -37,6 +38,11 @@ class StopDucc(DuccUtil): def stop_component(self, component, force): + if ( (component == 'broker') and self.automanage ): + print 'Stopping broker' + self.stop_broker() + return + # # If it's an unqualified management component, we need to get it's qualified name # @@ -177,7 +183,6 @@ class StopDucc(DuccUtil): print 'badarg', a self.invalid('bad arg: ' + a) - # 'management' means stop all the management daemons - if specific components are also specified # there is at least a redundancy and maybe also a conflict. if ( do_components and management ): @@ -186,13 +191,21 @@ class StopDucc(DuccUtil): # avaid confusion by insuring that if 'all', then nothing else is specified if ( all and ( do_components or management ) ): self.invalid("The --all option is mutually exclusive with --management and --component") - # 'all' means everything. we use broadcast. should use check_ducc to make sure # it actually worked, and find the stragglers. if ( all ): if ( not force ) : self.clean_shutdown() + + sleeptime = 5 + print "Waiting " + str(sleeptime) + " seconds to broadcast agent shutdown." + time.sleep(sleeptime) + + if ( self.automanage ): + print "Stopping broker" + self.stop_broker() + if ( os.path.exists(self.pid_file) ): os.remove(self.pid_file) return Modified: uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml (original) +++ uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml Tue Jan 22 16:22:39 2013 @@ -46,7 +46,8 @@ </systemUsage> <transportConnectors> - <transportConnector name="openwire" uri="tcp://0.0.0.0:61616?transport.soWriteTimeout=45000"/> + <!-- transportConnector name="openwire" uri="tcp://0.0.0.0:61616?transport.soWriteTimeout=45000"/ --> + <transportConnector name="openwire" uri="tcp://0.0.0.0:${DUCC_AMQPORT}?${DUCC_AMQ_DECORATIONS}"/> </transportConnectors> </broker> Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes (original) +++ uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes Tue Jan 22 16:22:39 2013 @@ -1,8 +1,9 @@ scheduling.class_set = background low normal high urgent weekly fixed reserve JobDriver scheduling.default.name = normal +scheduling.default.name.reserve = fixed -scheduling.nodepool = reserve -scheduling.nodepool.reserve = reserved.nodes +scheduling.nodepool = jobdriver +scheduling.nodepool.jobdriver = jobdriver.nodes # # The first five classes are blade-like unfair share @@ -64,14 +65,14 @@ scheduling.class.reserve.policy scheduling.class.reserve.priority = 1 scheduling.class.reserve.cap = 0 scheduling.class.reserve.enforce.memory = false -#scheduling.class.reserve.nodepool = reserve +scheduling.class.reserve.max_machines = 10 # # Job driver, always going to succeed # -scheduling.class.JobDriver.policy = RESERVE +scheduling.class.JobDriver.policy = FIXED_SHARE scheduling.class.JobDriver.priority = 0 scheduling.class.JobDriver.cap = 0 -#scheduling.class.JobDriver.nodepool = reserve scheduling.class.JobDriver.enforce.memory = false scheduling.class.JobDriver.max_machines = 5 +scheduling.class.JobDriver.nodepool = jobdriver Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties?rev=1437025&r1=1437024&r2=1437025&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties (original) +++ uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties Tue Jan 22 16:22:39 2013 @@ -1,35 +1,54 @@ # Declare which components to load into the jvm on process startup. - ducc.jms.provider=activemq # --------------------------------------------------- -# deprecated ducc.broker.url in favor of breaking a url into pieces -# as defined below -#ducc.broker.url=tcp://localhost:61616 # --------------------------------------------------- -# define broker protocol. Should typically be tcp. *** Dont specify : or // *** +# Define the ActivemQ broker configuration. +# ducc.broker.protocol - Ddefine broker protocol. Should typically be +# tcp. *** Dont specify : or // *** +# ducc.broker.hostname - The name of the host where the broker is running. +# ducc.broker.port - The broker's connection port +# ducc.broker.url.decoration - The broker *client* url decoration. +# recoveryInterval=30000 - Specifies an interval between recovery attempts, +# i.e when a connection is being refreshed, +# in milliseconds +# jms.useCompression=true - Compress message body +# ducc.broker.name - the broker name - must match the actual broker name +# in the broker config. THIS IS NOT THE BROKER HOSTNAME +# WHICH IS CONFIGURED IN ducc.broker.hostname. +# ducc.broker.jmx.port - the Broker's jmx port +# ducc.broker.protocol=tcp -# define hostname where the broker is running -ducc.broker.hostname=localhost -# define broker port +ducc.broker.hostname=${ducc.head} ducc.broker.port=61616 -# define broker url parms (url decoration). *** Dont specify leading ? *** -# -# recoveryInterval=30000 - Specifies an interval between recovery attempts, i.e. when -# a connection is being refreshed, in milliseconds -# jms.useCompression=true - Compress message body +ducc.broker.url.decoration=wireFormat.maxInactivityDuration=0&jms.useCompression=true +ducc.broker.name=localhost +ducc.broker.jmx.port=1099 + +# ActiveMQ Auto-management configuration +# docc.broker.automanage - if true, DUCC will start and stop the broker as needed. +# Otherwise, the installation is responsible for +# manageing ActiveMQ +# ducc.broker.memor.options - If automanaged, the memory configuration passed to the JVM +# ducc.broker.configuration - If automanaged, the location of the ActiveMQ broker configuration +# ducc.broker.home - If automanaged, the location of the ActiveMQ installation +# ducc.broker.server.url.decoration - If automanaged, the broker URL decoration # -ducc.broker.url.decoration=wireFormat.maxInactivityDuration=0&recoveryInterval=30000&jms.useCompression=true +ducc.broker.automanage = true +ducc.broker.memory.options = -Xmx2G +ducc.broker.configuration = conf/activemq-nojournal5.xml +ducc.broker.home = ${DUCC_HOME}/activemq +ducc.broker.server.url.decoration = transport.soWriteTimeout=45000 # To enable tracing of RM messages arriving in OR and NodeMetrics arriving in WS. #ducc.transport.trace = orchestrator:RmStateDuccEvent webserver:NodeMetricsUpdateDuccEvent -# define broker name. This name should match the actual broker name -# defined in activemq-nojournal5.xml. The name is used by JD to -# connect to a remote broker in order to remove its queue when -# JD is done processing -ducc.broker.name=localhost -ducc.broker.jmx.port=1099 +# +# This specifies the path and parameters needed to start activeMQ. The default starts +# the activeMQ that is embedded with DUCC. Change this if you wish to run your own +# activeMQ. Set to NONE if you do not want DUCC to start the broker when DUCC itself +# is started. +ducc.broker.startup = default amq starter ducc.cluster.name=Apache UIMA-DUCC @@ -49,6 +68,7 @@ ducc.node.min.swap.threshold=1000000 # Specify the jvm here. If not here, java must be in your path. If java is in neither place DUCC won't start. #ducc.jvm = /share/ibm-java-sr10-x86_64-60/bin/java +ducc.jvm = /share/jdk1.6/bin/java # administrative endpoint for all ducc components ducc.admin.endpoint=ducc.admin.channel @@ -110,8 +130,8 @@ ducc.jd.state.publish.rate=15000 ducc.jd.queue.prefix=ducc.jd.queue. ducc.jd.host.class=JobDriver ducc.jd.host.description=Job Driver -ducc.jd.host.memory.size=8GB -ducc.jd.host.number.of.machines=2 +ducc.jd.host.memory.size=4GB +ducc.jd.host.number.of.machines=1 ducc.jd.host.user=System # ========== Job Driver Configuration block ========== @@ -124,8 +144,8 @@ ducc.sm.meta.ping.rate = 60000 ducc.sm.meta.ping.stability = 10 ducc.sm.meta.ping.timeout = 500 ducc.sm.http.port=19989 -ducc.sm.http.node=localhost -ducc.sm.default.linger=60000 +ducc.sm.http.node=${ducc.head} +ducc.sm.default.linger=300000 # === END == Service Manager Configuration block ========== # ========== Orchestrator Configuration block ========== @@ -162,7 +182,7 @@ ducc.orchestrator.job.factory.classpath. ducc.orchestrator.http.port=19988 # !!!!!!!! Node where OR is running. This is needed by CLI # to compose a URL to access OR jetty server -ducc.orchestrator.node=localhost +ducc.orchestrator.node=${ducc.head} # ========== Orchestrator Configuration block ========== # Resource Manager Configuration block @@ -176,7 +196,7 @@ ducc.rm.state.publish.rate = 60000 # Amount of Dram to reserve before computing shares for a machine In GB ducc.rm.reserved.dram = 0 # Base size of dram quantum in Gb -ducc.rm.share.quantum = 15 +ducc.rm.share.quantum = 4 # Implementation class for actual scheduling algorithm #ducc.rm.scheduler = org.apache.uima.ducc.sm.pm.orchestrator.agent.jd.rm.rm.scheduler.ClassBasedScheduler ducc.rm.scheduler = org.apache.uima.ducc.rm.scheduler.NodepoolScheduler @@ -189,9 +209,9 @@ ducc.rm.default.memory = 15 #default number of threads, if not specified in job ducc.rm.default.threads = 4 # number of node metrics heartbeats to wait for before rm starts up -ducc.rm.init.stability = 3 +ducc.rm.init.stability = 2 # number of missed node metrics updates to consider node down -ducc.rm.node.stability = 3 +ducc.rm.node.stability = 5 # which policy to use when shrinking/evicting shares - alternatively, SHRINK_BY_MACHINE ducc.rm.eviction.policy = SHRINK_BY_INVESTMENT # max nodes to initially allocate until init is complete @@ -203,7 +223,7 @@ ducc.rm.expand.by.doubling = true # Predict when a job will end and avoid expanding if not needed ducc.rm.prediction = true # Add this fudge factor (milliseconds) to the expansion target when using prediction -ducc.rm.prediction.fudge = 10000 +ducc.rm.prediction.fudge = 120000 ducc.rm.defragmentation = true # What is minimum number of shares before we do defrag? ducc.rm.fragmentation.threshold = 2