Author: challngr
Date: Tue Jan 22 16:22:39 2013
New Revision: 1437025
URL: http://svn.apache.org/viewvc?rev=1437025&view=rev
Log:
UIMA-2584
Scripting to do post-installation tasks as described.
Updates to all scripting components to auto-manage the ActiveMQ server using
common configuration in ducc.properties.
Updates to default ducc.properties for the above.
Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (with props)
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim
uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Tue Jan 22 16:22:39
2013
@@ -40,6 +40,12 @@ class CheckDucc(DuccUtil):
def validate(self, checkdate):
verify_slave_node(checkdate, self.ducc_properties)
+ def verify_activemq(self):
+ if ( self.is_amq_active() ):
+ print 'ActiveMQ is found listening at', self.broker_protocol +
"://" + self.broker_host + ':' + self.broker_port
+ return True
+ return False
+
def usage(self, msg):
if ( msg != None ):
print msg
@@ -122,7 +128,6 @@ class CheckDucc(DuccUtil):
print 'badarg', a
usage('bad arg: ' + a)
-
if ( do_validate ):
# if validating, ONLY validate, called via ssh usually
self.validate(checkdate)
@@ -131,15 +136,17 @@ class CheckDucc(DuccUtil):
if ( reap and (user == 'ducc') ):
usage('Can only reap non-udcc users')
-
jvm = self.ducc_properties.get('ducc.jvm')
if ( jvm == None ):
- print 'WARN: "ducc.jvm is not specified in ducc.properties.
Default is simply "java" which may not work on all nodes.'
+ print 'WARN: ducc.jvm is not specified in ducc.properties. Default
is simply "java" which may not work on all nodes.'
if ( not verify_master_node(self.ducc_properties) ):
- print 'FAIL: Cannot run javac to build the java test'
+ print 'FAIL: Cannot verify master mode'
return
-
+
+ if ( not self.verify_activemq() ):
+ print 'ActiveMQ is not running on', self.broker_protocol + "://" +
self.broker_host + ':' + self.broker_port
+
# init the PID file
pids = DuccProperties()
pids.load_if_exists(self.pid_file)
@@ -238,6 +245,7 @@ class CheckDucc(DuccUtil):
return
if ( kill ):
+ self.stop_broker()
self.remove_orchestrator_lock()
if ( len(pids) == 0):
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc.py Tue Jan 22 16:22:39 2013
@@ -29,7 +29,29 @@ from ducc_util import DuccProperties
from local_hooks import verify_slave_node
class Ducc(DuccUtil):
+
+ def run_broker(self, component):
+ broker_port = self.ducc_properties.get('ducc.broker.port')
+ broker_url_decoration =
self.ducc_properties.get('ducc.broker.server.url.decoration')
+ broker_memory_opts =
self.ducc_properties.get('ducc.broker.memory.options')
+ broker_config = self.ducc_properties.get('ducc.broker.configuration')
+ broker_home = self.ducc_properties.get('ducc.broker.home')
+
+ if ( broker_config[0] != '/' ): # relative to broker_home if not
absolute
+ broker_config = broker_home + '/' + broker_config
+
+ os.environ['ACTIVEMQ_OPTS'] = '-DDUCC_AMQ_PORT=' + broker_port + '
-DDUCC_AMQ_DECORATION=' + broker_url_decoration
+ os.environ['ACTIVEMQ_OPTS_MEMORY'] = broker_memory_opts
+ os.environ['ACTIVEMQ_HOME'] = broker_home
+
+ here = os.getcwd()
+ os.chdir(broker_home + '/bin')
+ CMD = './activemq start xbean:' + broker_config
+ self.spawn(CMD)
+ os.chdir(here)
+ print "Started AMQ broker"
+
def run_component(self, component, or_parms, numagents, rmoverride,
background, nodup, localdate):
if ( component == 'all' ):
@@ -38,9 +60,14 @@ class Ducc(DuccUtil):
complist = component.split(',')
args = None
+ # ducc-head needs to be in system properties before the ducc daemon
reads ducc.properties
+ # to insure it can be substituted properly
+ ducc_head = self.ducc_properties.get('ducc.head')
+
jvm_opts = []
jvm_opts.append('-Dos.page.size=' + self.os_pagesize)
jvm_opts.append('-Dducc.deploy.configuration=' + self.DUCC_HOME +
'/resources/ducc.properties')
+ jvm_opts.append('-Dducc.head=' + ducc_head)
service = 'org.apache.uima.ducc.common.main.DuccService'
for c in complist:
@@ -163,7 +190,7 @@ class Ducc(DuccUtil):
print ' ducc.py -k'
print 'Where:'
print ' -c <component> is the name of the comp[onent to start,
currently one of'
- print ' agent rm sm pm ws orchestrator'
+ print ' agent rm sm pm ws orchestrator broker'
print ' -- or --'
print ' all - to start rm sm pm ws orchestrator'
print ' NOTE -- that agents should be started separately'
@@ -172,6 +199,9 @@ class Ducc(DuccUtil):
print ' -n <numagents> if > 1, multiple agents are started (testing
mode)'
print ' -o <mem-in-GB> rm memory override for use on small machines'
print ' -k causes the entire DUCC system to shutdown'
+ print ' --nodup If specified, do not start a process if it appears
to be already started.'
+ print ' --or_parms [cold|warm|hot]'
+ print ' --ducc_head nodename the name of the "ducc head" where ducc
is started from'
print ' arguments - any additional arguments to pass to the
component.'
sys.exit(1)
@@ -188,7 +218,7 @@ class Ducc(DuccUtil):
localdate = 0
try:
- opts, args = getopt.getopt(argv, 'bc:d:n:o:k?v', ['or_parms=',
'nodup'])
+ opts, args = getopt.getopt(argv, 'bc:d:n:o:k?v', ['or_parms=',
'nodup' ])
except:
self.usage('Bad arguments ' + ' '.join(argv))
@@ -196,8 +226,7 @@ class Ducc(DuccUtil):
if ( o == '-c' ) :
component = a
if ( component == 'or' ):
- component = 'orchestrator'
-
+ component = 'orchestrator'
elif ( o == '-b'):
background = True
elif ( o == '-d'):
@@ -217,18 +246,21 @@ class Ducc(DuccUtil):
else:
print 'badarg', a
usage('bad arg: ' + a)
-
+
if ( shutdown ):
if ( component != None ):
print 'Note: -c flag for component not allowed when shutting
down. Shutdown aborted'
sys.exit(1);
self.clean_shutdown();
sys.exit(1)
-
+
if ( component == None ):
self.usage("Must specify component")
- self.run_component(component, or_parms, numagents, rmoverride,
background, nodup, localdate)
+ if ( component == 'broker' ):
+ self.run_broker(background)
+ else:
+ self.run_component(component, or_parms, numagents, rmoverride,
background, nodup, localdate)
return
def __call__(self, *args):
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1437025&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install Tue Jan 22
16:22:39 2013
@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+import shutil
+import subprocess
+
+from ducc_boot import *
+set_ducc_home()
+
+from ducc_util import DuccUtil
+from ducc_util import DuccProperties
+from ducc import Ducc
+
+class PostInstall(DuccUtil):
+
+ def usage(self, msg):
+
+ if ( msg != None ):
+ print ' '.join(msg)
+
+
+ print "Usage:"
+ print " stop_sim [options]"
+ print " If no options are given, all ducc processes from
sim.pids are stopped using SIGINT"
+ print ""
+ print "Options:"
+ print " -c --component compenent name"
+ print " The name is one of", self.default_components, "or an ID
from sim.pids or the nodelist"
+ print " that was used to start DUCC"
+ print ""
+ print " -i --instance instancid"
+ print " Sends SIGINT to the indicated process, enabling orderly
shutdown."
+ print ""
+ print " -k --kill"
+ print " Sends SIGKILL to the indicated process"
+ print ""
+ print " -p --pause"
+ print " Sends SIGSTOP to the indicated process"
+ print ""
+ print " -r --resume"
+ print " Sends SIGCONT to the indicated process"
+ print ""
+ print " -v, --version"
+ print " Print the current DUCC version"
+ print ""
+ sys.exit(1)
+
+
+ def fail(self, *msg):
+ print ' '.join(msg)
+ print "POST INSTALLATION FAILED"
+ sys.exit(1)
+
+
+ def warn(self, *msg):
+ print ''
+ print 'WARNING'
+ print 'WARNing', ' '.join(msg)
+ print 'WARNING'
+ print ''
+
+ def create_keystore(self, keytool):
+
+ '''
+ CN - Common Name of the certificate owner
+ OU - Organizational Unit of the certificate owner
+ O - Organization to which the certificate owner belongs
+ L - Locality name of the certificate owner
+ S - State or province of the certificate owner
+ C - Country of the certificate owner
+ '''
+
+ keystore = self.DUCC_HOME + "/webserver/etc/keystore"
+ cmd = 'rm ' + keystore
+ os.system(cmd);
+
+ pw = self.ducc_properties.get('ducc.ws.port.ssl.pw')
+ if ( pw == 'quackquack' ):
+ print self.warn('The default ssl pw of ', pw, ' is configured. For
security, change the property \n\tducc.ws.port.ssl.pw to a secure password in
ducc.properties.')
+
+
+ #/usr/bin/keytool
+ cmd = self.keytool
+ cmd += ' '
+ cmd += '-genkey'
+ cmd += ' '
+ cmd += '-noprompt'
+ cmd += ' '
+ cmd += '-alias jetty'
+ cmd += ' '
+ cmd += '-dname "CN=org.apache.uima.ducc, OU=uima.ducc, O=Apache,
L=Wilmington, S=Delaware, C=USA"'
+ cmd += ' '
+ cmd += '-keyalg RSA'
+ cmd += ' '
+ cmd += '-validity 10000'
+ cmd += ' '
+ cmd += '-keystore ' + keystore
+ cmd += ' '
+ cmd += '-storepass '+pw
+ cmd += ' '
+ cmd += '-keypass '+pw
+ os.system(cmd);
+ print 'keystore created.'
+
+
+ # Setup and verify amq
+ # make sure verify_ducc is sufficient - maybe move some checks to there?
+
+ def get_java_bindir(self):
+
+ java = self.java()
+ if ( java == None ):
+ self.fail("THe path to java must be configured in ducc.properties
as property", 'ducc.jvm')
+
+ if ( not java.startswith('/') ):
+ self.fail("THe java path in property ducc.jvm must be absolute.")
+
+ # We're going to do more checks here so we don't proceed with
bogosities
+ proc = subprocess.Popen(java + ' -version', shell=True, bufsize=0,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ lines = []
+ for line in proc.stdout:
+ lines.append(line.strip())
+ proc.wait()
+ rc = proc.returncode
+
+ for line in lines:
+ print "JAVA: " + line
+
+ vertoks = lines[0].split()
+ self.java_version = vertoks[-1]
+
+ if ( rc != 0 ):
+ self.fail("Configured java at '", java, "' is incorrect.")
+
+ return os.path.dirname(java)
+
+ def get_java_version(self):
+ return self.java_version
+
+ def get_java_keytool(self, bindir):
+ keytool = bindir + "/keytool"
+ if ( not os.path.exists(keytool) ):
+ self.fail("Cannot find keytool in ", bindir + '.', + "Is ducc.jvm
configured correctly?")
+ return keytool
+
+ def check_nodes(self):
+ nodes = self.DUCC_HOME + "/resources/ducc.nodes"
+ if ( not os.path.exists(nodes) ):
+ nf = open(nodes, 'w')
+ nf.write(self.localhost)
+ nf.close()
+ print "Initial", nodes, "created."
+ else:
+ print "Verified existence of default nodefile " + nodes
+ print ''
+
+ jdnodes = self.DUCC_HOME + "/resources/jobdriver.nodes"
+ if ( not os.path.exists(jdnodes) ):
+ rn = open(jdnodes, "w")
+ rn.write(self.localhost)
+ rn.close()
+ print "Initial nodepool", jdnodes, "created.\n"
+ else:
+ print "Verified existence of default JobDriver nodepool " + jdnodes
+ print ''
+
+
+ def check_activemq(self):
+ if ( self.automanage ): # the default
+ print 'ActiveMQ is automanaged on node ', self.broker_url
+ else:
+ print 'ActiveMQ is managed by the installation and is not
verified.'
+
+ def setup_ducc_head(self):
+ ducc_head = self.ducc_properties.get('ducc.head')
+ if ( ducc_head == None ):
+ ducc_head = self.localhost
+ self.ducc_properties.put('ducc.head', ducc_head)
+ propsfile = self.DUCC_HOME + "/resources/ducc.properties"
+ tmp_propsfile = propsfile + ".tmp"
+ f = open(propsfile)
+ tf = open(tmp_propsfile, "w")
+ tf.write("# ducc.head is the node where the main DUCC daemons
run\n")
+ tf.write("ducc.head = " + self.localhost + "\n\n")
+ for line in f:
+ tf.write(line);
+ f.close()
+ tf.close()
+ os.rename(propsfile, propsfile + ".orig")
+ os.rename(tmp_propsfile, propsfile)
+ self.read_properties() # force re-eval with ducc.head
+
+ print "Ducc head is configured as", ducc_head, '\n'
+
+ def main(self, args):
+
+ print 'Python version:'
+ print sys.version
+ print ''
+
+ py_version = sys.version_info
+ if ( (py_version[0] != 2) or (py_version[1] < 4) ):
+ self.fail("Python must be installed at level 2.4 or higher.")
+
+ self.setup_ducc_head()
+ self.check_nodes()
+ self.check_activemq()
+
+ # insure java is configured and installed
+ self.java_bindir = self.get_java_bindir()
+ self.keytool = self.get_java_keytool(self.java_bindir)
+ print 'Java version:', self.get_java_version()
+ if ( not '1.6' in self.get_java_version() ):
+ print 'WARNING: DUCC is only tested under Java 1.6'
+ else:
+ print 'Java is verified.'
+ print ''
+
+ self.create_keystore(self.keytool)
+ print 'Web server keystore generated from ducc.properties'
+ print ''
+
+ # Make duccling
+ here = os.getcwd()
+ os.chdir("../duccling/src")
+ os.system("make clean all")
+ os.chdir(here)
+ shutil.copyfile("../duccling/src/ducc_ling", self.DUCC_HOME +
"/admin/ducc_ling")
+ print 'Initial ducc_ling is installed. See the installation guide for
multi-user setup.'
+
+if __name__ == "__main__":
+ postinstall = PostInstall()
+ postinstall.main(sys.argv[1:])
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_post_install
------------------------------------------------------------------------------
svn:executable = *
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Jan 22
16:22:39 2013
@@ -43,22 +43,32 @@ class DuccProperties:
self.props = {}
#
- # Expand ${} values from env or from this properties file itself
+ # Expand all ${} values from env or from this properties file itself
+ # The search order is:
+ # 1 look in this properties file
+ # 2 look in the environment
#
def do_subst(self, str):
-
key = None
- p = re.compile("\\$\\{[a-zA-Z0-9_]+\\}")
- m = p.match(str)
+ p = re.compile("\\$\\{[a-zA-Z0-9_\\.\\-]+\\}")
+ ndx = 0
- if ( m != None ):
+ response = str
+ m = p.search(response, ndx)
+ while ( m != None ):
key = m.group()[2:-1]
- #print str, m, m.group(), key
- val = os.environ[key]
- response = string.replace(str, m.group() , val)
- else:
- response = str
+ val = None
+ if ( self.has_key(key) ):
+ val = self.get(key)
+ elif ( os.environ.has_key(key) ):
+ val = os.environ[key]
+
+ if ( val != None ):
+ response = string.replace(response, m.group() , val)
+ ndx = m.start()+1
+ m = p.search(response, ndx)
+
return response
def mkitem(self, line):
@@ -77,7 +87,7 @@ class DuccProperties:
key = line[:mobj.start()].strip()
val = line[mobj.end():].strip()
#print 'NEXT', mobj.start(), 'END', mobj.end(), 'KEY', key, 'VAL',
val
- val = self.do_subst(val)
+ # val = self.do_subst(val) # we'll do lazy subst on get instead
self.props[key] = val
else:
self.props[line] = None
@@ -121,7 +131,7 @@ class DuccProperties:
#
def get(self, key):
if ( self.props.has_key(key) ):
- return self.props[key]
+ return self.do_subst(self.props[key]) # we'll do lazy subst on
get instead
return None
#
@@ -183,12 +193,13 @@ class DuccUtil:
self.broker_jmx_port =
self.ducc_properties.get('ducc.broker.jmx.port')
self.broker_decoration =
self.ducc_properties.get('ducc.broker.url.decoration')
self.broker_url = self.broker_protocol + '://' +
self.broker_host + ':' + self.broker_port
- self.agent_jvm_args =
self.ducc_properties.get('ducc.agent.jvm.args')
- self.ws_jvm_args =
self.ducc_properties.get('ducc.ws.jvm.args')
- self.pm_jvm_args =
self.ducc_properties.get('ducc.pm.jvm.args')
- self.rm_jvm_args =
self.ducc_properties.get('ducc.rm.jvm.args')
- self.sm_jvm_args =
self.ducc_properties.get('ducc.sm.jvm.args')
- self.or_jvm_args =
self.ducc_properties.get('ducc.orchestrator.jvm.args')
+ self.agent_jvm_args =
self.ducc_properties.get('ducc.agent.jvm.args')
+ self.ws_jvm_args = self.ducc_properties.get('ducc.ws.jvm.args')
+ self.pm_jvm_args = self.ducc_properties.get('ducc.pm.jvm.args')
+ self.rm_jvm_args = self.ducc_properties.get('ducc.rm.jvm.args')
+ self.sm_jvm_args = self.ducc_properties.get('ducc.sm.jvm.args')
+ self.or_jvm_args =
self.ducc_properties.get('ducc.orchestrator.jvm.args')
+
if ( self.broker_decoration == '' ):
self.broker_decoration = None
@@ -200,10 +211,7 @@ class DuccUtil:
self.webserver_node = self.localhost
def java(self):
- if ( self.jvm == None ):
- return 'java'
- else:
- return self.jvm
+ return self.jvm
def is_amq_active(self):
lines = self.popen('ssh', self.broker_host, 'netstat -an')
@@ -220,6 +228,16 @@ class DuccUtil:
return True
return False
+ def stop_broker(self):
+ broker_host = self.ducc_properties.get('ducc.broker.hostname')
+ broker_home = self.ducc_properties.get('ducc.broker.home')
+ here = os.getcwd()
+ CMD = broker_home + '/bin/activemq'
+ CMD = CMD + ' stop'
+ print CMD
+ self.ssh(broker_host, False, CMD)
+ pass
+
def version(self):
lines = self.popen(self.jvm, ' org.apache.uima.ducc.utils.Version')
line = lines.readline().strip()
@@ -575,7 +593,8 @@ class DuccUtil:
def clean_shutdown(self):
DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME +
"/resources/ducc.properties "
- self.spawn('java', DUCC_JVM_OPTS,
'org.apache.uima.ducc.common.main.DuccAdmin', '--killAll')
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.head=' +
self.ducc_properties.get('ducc.head')
+ self.spawn(self.java(), DUCC_JVM_OPTS,
'org.apache.uima.ducc.common.main.DuccAdmin', '--killAll')
def get_os_pagesize(self):
lines = self.popen('/usr/bin/getconf', 'PAGESIZE')
@@ -702,15 +721,21 @@ class DuccUtil:
self.broker_protocol = 'tcp'
self.broker_host = 'localhost'
self.broker_port = '61616'
- self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'viz']
+ self.default_components = ['rm', 'pm', 'sm', 'or', 'ws', 'viz',
'broker']
self.default_nodefiles = [self.DUCC_HOME + '/resources/ducc.nodes']
self.propsfile = self.DUCC_HOME + '/resources/ducc.properties'
self.localhost = os.uname()[1]
+ os.environ['NodeName'] = self.localhost # to match java code's
implicit propery so script and java match
self.pid_file = self.DUCC_HOME + '/state/ducc.pids'
self.set_classpath()
self.read_properties()
self.os_pagesize = self.get_os_pagesize()
+ manage_broker = self.ducc_properties.get('ducc.broker.automanage')
+ self.automanage = False
+ if (manage_broker in ('t', 'true', 'T', 'True')) :
+ self.automanage = True
+
if __name__ == "__main__":
util = DuccUtil()
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Tue Jan 22 16:22:39
2013
@@ -21,7 +21,7 @@
import os
import sys
-from time import time
+import time
from ducc_boot import *
set_ducc_home()
@@ -35,6 +35,29 @@ from local_hooks import verify_master_no
class StartDucc(DuccUtil):
+ def start_broker(self):
+
+ broker_host = self.ducc_properties.get('ducc.broker.hostname')
+ print 'broker host', broker_host
+ lines = self.ssh(broker_host, True, "'", self.DUCC_HOME +
'/admin/ducc.py', '-c', 'broker', "'")
+ while 1:
+ line = lines.readline().strip()
+ if ( not line ):
+ break
+ print '[] ' + line
+ if ( line.startswith('PID') ):
+ toks = line.split(' ') # get the PID
+ print "Broker on", broker_host, 'PID', toks[1]
+ self.pids.put('broker@' + broker_host, toks[1])
+ lines.close()
+ break
+
+ for i in range(0, 9):
+ if ( self.is_amq_active() ):
+ return
+ print 'Waiting for broker', str(i)
+ time.sleep(1)
+
def start_component(self, ducc, component, or_parms):
node = 'local'
@@ -61,7 +84,7 @@ class StartDucc(DuccUtil):
if ( node == 'local' ):
node = self.localhost
- lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py',
'-c', com, '-b', or_parms, '-d', str(time()), '--nodup' "'")
+ lines = self.ssh(node, True, "'", self.DUCC_HOME + '/admin/ducc.py',
'-c', com, '-b', or_parms, '-d', str(time.time()), '--nodup', "'")
# we'll capture anything that the python shell spews because it may be
useful, and then drop the
# pipe when we see a PID message
while 1:
@@ -89,7 +112,7 @@ class StartDucc(DuccUtil):
spacer = ' '
print host
- lines = self.ssh(host, True, "'", self.DUCC_HOME +
'/admin/ducc.py', '-c' 'agent', '-b', '-d', str(time()), '--nodup', "'")
+ lines = self.ssh(host, True, "'", self.DUCC_HOME +
'/admin/ducc.py', '-c' 'agent', '-b', '-d', str(time.time()), '--nodup', "'")
while 1:
line = lines.readline().strip()
#print '[]' + line
@@ -120,7 +143,6 @@ class StartDucc(DuccUtil):
else:
print spacer, line
-
def usage(self, *msg):
if ( msg[0] != None ):
print ' '.join(msg)
@@ -219,7 +241,6 @@ class StartDucc(DuccUtil):
else:
self.invalid('bad args: ', ' '.join(argv))
-
# 'management' means start all the management daemons - if specific
components are also specified
# there is at least a redundancy and maybe also a conflict.
if ( (len(components) != 0) and management ):
@@ -257,6 +278,12 @@ class StartDucc(DuccUtil):
sys.exit(1)
# activeMQ needs to be started externally before starting any DUCC
processes
+ if ( self.automanage and ('broker' in components) ):
+ if ( self.is_amq_active() ):
+ print 'ActiveMQ is already running on host and port:',
self.broker_host + ':' + self.broker_port, 'NOT restarting'
+ else:
+ self.start_broker()
+
#print
'A--------------------------------------------------------------------------------'
if ( self.is_amq_active() ):
print 'ActiveMQ is found on configured host and port:',
self.broker_host + ':' + self.broker_port
@@ -271,7 +298,10 @@ class StartDucc(DuccUtil):
if ( len(components) != 0 ):
print 'Starting', or_parms
for com in components:
- self.start_component(ducc, com, or_parms)
+ if ( com == 'broker' ):
+ pass # already started
+ else:
+ self.start_component(ducc, com, or_parms)
else:
print 'Not starting management components.'
#print
'C--------------------------------------------------------------------------------'
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_sim Tue Jan 22 16:22:39
2013
@@ -21,6 +21,7 @@
import os
import sys
+import time
import getopt
from ducc_boot import *
@@ -32,6 +33,29 @@ from ducc import Ducc
class StartSim(DuccUtil):
+ def start_broker(self):
+
+ broker_host = self.ducc_properties.get('ducc.broker.hostname')
+ print 'broker host', broker_host
+ lines = self.ssh(broker_host, True, "'", self.DUCC_HOME +
'/admin/ducc.py', '-c', 'broker', "'")
+ while 1:
+ line = lines.readline().strip()
+ if ( not line ):
+ break
+ print '[] ' + line
+ if ( line.startswith('PID') ):
+ toks = line.split(' ') # get the PID
+ print "Broker on", broker_host, 'PID', toks[1]
+ self.pids.put('broker@' + broker_host, toks[1])
+ lines.close()
+ break
+
+ for i in range(0, 9):
+ if ( self.is_amq_active() ):
+ return
+ print 'Waiting for broker', str(i)
+ time.sleep(1)
+
def run_local_agent(self, pnode, ip, memory ):
memory = int(memory) * 1024 * 1024 # to GB from KB
@@ -56,7 +80,12 @@ class StartSim(DuccUtil):
# Start admin components rm pm sm ws or, on local node using Ducc.py
#
def startComponents(self, components, or_parms):
+
for (com, com) in components.items():
+
+ if ( com == 'broker' ):
+ continue
+
if ( com in ('ws', 'viz') ):
node = self.webserver_node
else:
@@ -279,8 +308,12 @@ class StartSim(DuccUtil):
if ( (IP != None) or (memory != None) or ( pseudoname != None )) :
self.invalid("Running with a nodelist is not compatible with
running a single agent.");
+ if ( components.get('broker') != None ):
+ self.start_broker()
+
if ( len(nodefiles) != 0 ):
self.startAgents(nodefiles, instances)
+
self.startComponents(components, or_parms)
self.pids.write('sim.pids')
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/stop_ducc Tue Jan 22 16:22:39
2013
@@ -21,6 +21,7 @@
import os
import sys
+import time
from ducc_boot import *
set_ducc_home()
@@ -37,6 +38,11 @@ class StopDucc(DuccUtil):
def stop_component(self, component, force):
+ if ( (component == 'broker') and self.automanage ):
+ print 'Stopping broker'
+ self.stop_broker()
+ return
+
#
# If it's an unqualified management component, we need to get it's
qualified name
#
@@ -177,7 +183,6 @@ class StopDucc(DuccUtil):
print 'badarg', a
self.invalid('bad arg: ' + a)
-
# 'management' means stop all the management daemons - if specific
components are also specified
# there is at least a redundancy and maybe also a conflict.
if ( do_components and management ):
@@ -186,13 +191,21 @@ class StopDucc(DuccUtil):
# avaid confusion by insuring that if 'all', then nothing else is
specified
if ( all and ( do_components or management ) ):
self.invalid("The --all option is mutually exclusive with
--management and --component")
-
# 'all' means everything. we use broadcast. should use check_ducc to
make sure
# it actually worked, and find the stragglers.
if ( all ):
if ( not force ) :
self.clean_shutdown()
+
+ sleeptime = 5
+ print "Waiting " + str(sleeptime) + " seconds to broadcast
agent shutdown."
+ time.sleep(sleeptime)
+
+ if ( self.automanage ):
+ print "Stopping broker"
+ self.stop_broker()
+
if ( os.path.exists(self.pid_file) ):
os.remove(self.pid_file)
return
Modified: uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml
(original)
+++ uima/sandbox/uima-ducc/trunk/src/main/config/activemq-nojournal5.xml Tue
Jan 22 16:22:39 2013
@@ -46,7 +46,8 @@
</systemUsage>
<transportConnectors>
- <transportConnector name="openwire"
uri="tcp://0.0.0.0:61616?transport.soWriteTimeout=45000"/>
+ <!-- transportConnector name="openwire"
uri="tcp://0.0.0.0:61616?transport.soWriteTimeout=45000"/ -->
+ <transportConnector name="openwire"
uri="tcp://0.0.0.0:${DUCC_AMQPORT}?${DUCC_AMQ_DECORATIONS}"/>
</transportConnectors>
</broker>
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.classes Tue Jan 22
16:22:39 2013
@@ -1,8 +1,9 @@
scheduling.class_set = background low normal high urgent weekly fixed
reserve JobDriver
scheduling.default.name = normal
+scheduling.default.name.reserve = fixed
-scheduling.nodepool = reserve
-scheduling.nodepool.reserve = reserved.nodes
+scheduling.nodepool = jobdriver
+scheduling.nodepool.jobdriver = jobdriver.nodes
#
# The first five classes are blade-like unfair share
@@ -64,14 +65,14 @@ scheduling.class.reserve.policy
scheduling.class.reserve.priority = 1
scheduling.class.reserve.cap = 0
scheduling.class.reserve.enforce.memory = false
-#scheduling.class.reserve.nodepool = reserve
+scheduling.class.reserve.max_machines = 10
#
# Job driver, always going to succeed
#
-scheduling.class.JobDriver.policy = RESERVE
+scheduling.class.JobDriver.policy = FIXED_SHARE
scheduling.class.JobDriver.priority = 0
scheduling.class.JobDriver.cap = 0
-#scheduling.class.JobDriver.nodepool = reserve
scheduling.class.JobDriver.enforce.memory = false
scheduling.class.JobDriver.max_machines = 5
+scheduling.class.JobDriver.nodepool = jobdriver
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties?rev=1437025&r1=1437024&r2=1437025&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties Tue Jan 22
16:22:39 2013
@@ -1,35 +1,54 @@
# Declare which components to load into the jvm on process startup.
-
ducc.jms.provider=activemq
# ---------------------------------------------------
-# deprecated ducc.broker.url in favor of breaking a url into pieces
-# as defined below
-#ducc.broker.url=tcp://localhost:61616
# ---------------------------------------------------
-# define broker protocol. Should typically be tcp. *** Dont specify : or // ***
+# Define the ActivemQ broker configuration.
+# ducc.broker.protocol - Ddefine broker protocol. Should typically be
+# tcp. *** Dont specify : or // ***
+# ducc.broker.hostname - The name of the host where the broker is running.
+# ducc.broker.port - The broker's connection port
+# ducc.broker.url.decoration - The broker *client* url decoration.
+# recoveryInterval=30000 - Specifies an interval between recovery
attempts,
+# i.e when a connection is being refreshed,
+# in milliseconds
+# jms.useCompression=true - Compress message body
+# ducc.broker.name - the broker name - must match the actual broker name
+# in the broker config. THIS IS NOT THE BROKER
HOSTNAME
+# WHICH IS CONFIGURED IN ducc.broker.hostname.
+# ducc.broker.jmx.port - the Broker's jmx port
+#
ducc.broker.protocol=tcp
-# define hostname where the broker is running
-ducc.broker.hostname=localhost
-# define broker port
+ducc.broker.hostname=${ducc.head}
ducc.broker.port=61616
-# define broker url parms (url decoration). *** Dont specify leading ? ***
-#
-# recoveryInterval=30000 - Specifies an interval between recovery attempts,
i.e. when
-# a connection is being refreshed, in milliseconds
-# jms.useCompression=true - Compress message body
+ducc.broker.url.decoration=wireFormat.maxInactivityDuration=0&jms.useCompression=true
+ducc.broker.name=localhost
+ducc.broker.jmx.port=1099
+
+# ActiveMQ Auto-management configuration
+# docc.broker.automanage - if true, DUCC will start and stop the broker
as needed.
+# Otherwise, the installation is responsible for
+# manageing ActiveMQ
+# ducc.broker.memor.options - If automanaged, the memory configuration
passed to the JVM
+# ducc.broker.configuration - If automanaged, the location of the ActiveMQ
broker configuration
+# ducc.broker.home - If automanaged, the location of the ActiveMQ
installation
+# ducc.broker.server.url.decoration - If automanaged, the broker URL
decoration
#
-ducc.broker.url.decoration=wireFormat.maxInactivityDuration=0&recoveryInterval=30000&jms.useCompression=true
+ducc.broker.automanage = true
+ducc.broker.memory.options = -Xmx2G
+ducc.broker.configuration = conf/activemq-nojournal5.xml
+ducc.broker.home = ${DUCC_HOME}/activemq
+ducc.broker.server.url.decoration = transport.soWriteTimeout=45000
# To enable tracing of RM messages arriving in OR and NodeMetrics arriving in
WS.
#ducc.transport.trace = orchestrator:RmStateDuccEvent
webserver:NodeMetricsUpdateDuccEvent
-# define broker name. This name should match the actual broker name
-# defined in activemq-nojournal5.xml. The name is used by JD to
-# connect to a remote broker in order to remove its queue when
-# JD is done processing
-ducc.broker.name=localhost
-ducc.broker.jmx.port=1099
+#
+# This specifies the path and parameters needed to start activeMQ. The
default starts
+# the activeMQ that is embedded with DUCC. Change this if you wish to run your
own
+# activeMQ. Set to NONE if you do not want DUCC to start the broker when DUCC
itself
+# is started.
+ducc.broker.startup = default amq starter
ducc.cluster.name=Apache UIMA-DUCC
@@ -49,6 +68,7 @@ ducc.node.min.swap.threshold=1000000
# Specify the jvm here. If not here, java must be in your path. If java is
in neither place DUCC won't start.
#ducc.jvm = /share/ibm-java-sr10-x86_64-60/bin/java
+ducc.jvm = /share/jdk1.6/bin/java
# administrative endpoint for all ducc components
ducc.admin.endpoint=ducc.admin.channel
@@ -110,8 +130,8 @@ ducc.jd.state.publish.rate=15000
ducc.jd.queue.prefix=ducc.jd.queue.
ducc.jd.host.class=JobDriver
ducc.jd.host.description=Job Driver
-ducc.jd.host.memory.size=8GB
-ducc.jd.host.number.of.machines=2
+ducc.jd.host.memory.size=4GB
+ducc.jd.host.number.of.machines=1
ducc.jd.host.user=System
# ========== Job Driver Configuration block ==========
@@ -124,8 +144,8 @@ ducc.sm.meta.ping.rate = 60000
ducc.sm.meta.ping.stability = 10
ducc.sm.meta.ping.timeout = 500
ducc.sm.http.port=19989
-ducc.sm.http.node=localhost
-ducc.sm.default.linger=60000
+ducc.sm.http.node=${ducc.head}
+ducc.sm.default.linger=300000
# === END == Service Manager Configuration block ==========
# ========== Orchestrator Configuration block ==========
@@ -162,7 +182,7 @@ ducc.orchestrator.job.factory.classpath.
ducc.orchestrator.http.port=19988
# !!!!!!!! Node where OR is running. This is needed by CLI
# to compose a URL to access OR jetty server
-ducc.orchestrator.node=localhost
+ducc.orchestrator.node=${ducc.head}
# ========== Orchestrator Configuration block ==========
# Resource Manager Configuration block
@@ -176,7 +196,7 @@ ducc.rm.state.publish.rate = 60000
# Amount of Dram to reserve before computing shares for a machine In GB
ducc.rm.reserved.dram = 0
# Base size of dram quantum in Gb
-ducc.rm.share.quantum = 15
+ducc.rm.share.quantum = 4
# Implementation class for actual scheduling algorithm
#ducc.rm.scheduler =
org.apache.uima.ducc.sm.pm.orchestrator.agent.jd.rm.rm.scheduler.ClassBasedScheduler
ducc.rm.scheduler = org.apache.uima.ducc.rm.scheduler.NodepoolScheduler
@@ -189,9 +209,9 @@ ducc.rm.default.memory = 15
#default number of threads, if not specified in job
ducc.rm.default.threads = 4
# number of node metrics heartbeats to wait for before rm starts up
-ducc.rm.init.stability = 3
+ducc.rm.init.stability = 2
# number of missed node metrics updates to consider node down
-ducc.rm.node.stability = 3
+ducc.rm.node.stability = 5
# which policy to use when shrinking/evicting shares - alternatively,
SHRINK_BY_MACHINE
ducc.rm.eviction.policy = SHRINK_BY_INVESTMENT
# max nodes to initially allocate until init is complete
@@ -203,7 +223,7 @@ ducc.rm.expand.by.doubling = true
# Predict when a job will end and avoid expanding if not needed
ducc.rm.prediction = true
# Add this fudge factor (milliseconds) to the expansion target when using
prediction
-ducc.rm.prediction.fudge = 10000
+ducc.rm.prediction.fudge = 120000
ducc.rm.defragmentation = true
# What is minimum number of shares before we do defrag?
ducc.rm.fragmentation.threshold = 2
