Author: cwiklik
Date: Mon Oct 26 18:31:05 2015
New Revision: 1710670
URL: http://svn.apache.org/viewvc?rev=1710670&view=rev
Log:
uima-ducc-2.0.1 release
Added:
uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/ducc_update (with
props)
uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.css (with
props)
uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.pdf (with
props)
uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/installation.html
(with props)
Added: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/ducc_update
URL:
http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/ducc_update?rev=1710670&view=auto
==============================================================================
--- uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/ducc_update (added)
+++ uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/ducc_update Mon Oct 26
18:31:05 2015
@@ -0,0 +1,426 @@
+#! /usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+#
-------------------------------------------------------------------------------------
+# Updates an existing ducc installation (1.1.0 or newer) from a binary tar.gz
build
+# Updates in place so preserves all the site-specific files
+# Archives the current build in a timestamped direcory under
DUCC_HOME/../ducc_archives
+# - checks that ducc is not running
+# - creates a site.ducc.properties file if updating from DUCC 1.1.0
+# - creates a time-stamped archive directory to hold the current build
+# - archives and then updates existing directories,
+# - keeps the directories: logs state history
+# - keeps any other files in directories: resources lib
+# - keeps the customized resources files: site.ducc.properties
ducc.classes ducc.nodes jobdriver.nodes ducc.administrators
+# - rebuilds the non-privileged ducc_ling
+#
+# Note: any files added to directories other than resources & lib will not be
retained, but will be archived
+#
+# To revert back to an archived build:
+# - copy/move all of the archived files back to the runtime
+#
-------------------------------------------------------------------------------------
+import os
+import sys
+import datetime
+import fnmatch
+import re
+import shutil
+
+def usage():
+ print "Usage: ducc_update ducc-runtime ducc-binary-tar"
+ print ""
+ print " Updates the DUCC installed at 'ducc-runtime' with the DUCC build
in 'ducc-binary-tar'"
+ print " - checks that DUCC is not running"
+ print " - creates a site.ducc.properties file if updating from DUCC
1.1.0"
+ print " - creates a time-stamped archive directory to hold the current
build"
+ print " - archives and then updates existing directories,"
+ print " - keeps the directories: logs state history"
+ print " - keeps any other files in directories: resources lib"
+ print " - keeps the customized resources files: site.ducc.properties
ducc.classes ducc.nodes jobdriver.nodes ducc.administrators"
+ print " - rebuilds the non-privileged ducc_ling"
+ print " Note: any files added to directories other than resources & lib
will not be retained, but will be archived"
+ print ""
+ print " To revert back to an archived build:"
+ print " - copy/move all of the archived files back to the runtime"
+
+#-----------------------------------------------------------------------------------------
+# Get the version of the build from the name of the cli jar
+#-----------------------------------------------------------------------------------------
+def get_oversion(runtime):
+ fns = fnmatch.filter(os.listdir(runtime + '/lib/uima-ducc'),
'uima-ducc-cli-*.jar')
+ if ( len(fns) == 0 ):
+ raise Exception("Not a valid DUCC installation - missing versioned cli
jar")
+ m = re.match('uima\\-ducc\\-cli\\-([a-zA-Z0-9_\\.\\-]+)\\.jar', fns[0])
+ if ( m == None ):
+ raise Exception("Not a valid DUCC installation - invalid name: " +
fns[0])
+ return m.group(1)
+
+#-----------------------------------------------------------------------------------------
+# Get the version of the build from the name of the tar file
+#-----------------------------------------------------------------------------------------
+def get_nversion(tarfile):
+ p = re.compile('^uima\\-ducc\\-([a-zA-Z0-9_\\.\\-]+)\\-bin\\.tar\\.gz$')
+ m = p.search(tarfile, 0)
+ if ( m == None ):
+ raise Exception("Invalid tar file name: " + tarfile + " ... expecting:
uima-ducc-<version>-bin.tar.gz")
+ return m.group(1)
+
+#-----------------------------------------------------------------------------------------
+# Clear out the old expanded tarball if needed, and expand the new one
+#-----------------------------------------------------------------------------------------
+def expand_tarball(tarfile, nversion, outdir):
+ extract = os.path.join(outdir, 'apache-uima-ducc-' + nversion)
+ try:
+ if ( os.path.exists(extract) ):
+ shutil.rmtree(extract);
+ except:
+ raise Exception("Cannot remove old tarball extract: " + extract)
+
+ cmd = 'umask 022 && tar -C ' + outdir + ' -xf ' + tarfile
+ rc = os.system(cmd);
+ if ( rc != 0 ):
+ raise Exception("Command fails, rc= " + str(rc) + "\n " + cmd)
+
+ if not os.path.exists(extract):
+ raise Exception("Cannot extract runtime from tarball " + tarfile + "
Expecting to find " + extract)
+
+ return extract
+
+#-----------------------------------------------------------------------------------------
+# Convert final comments into a DUCC 1.1.0 properties file
+#-----------------------------------------------------------------------------------------
+def create_110_properties(source, dest):
+ found = False
+ with open(source) as f:
+ for line in f:
+ if not found:
+ if line.startswith("#=====MARKER====="):
+ found = True
+ outf = open(dest, 'w')
+ else:
+ outf.write('ducc.' + line[1:])
+ outf.close();
+
+#-----------------------------------------------------------------------------------------
+# Update directory by archiving all files that are replaced, unless in exclude
list
+# Optionally rename new files that don't replace existing files
+#-----------------------------------------------------------------------------------------
+def update_dir(olddir, newdir, archive, excludeFiles, rename):
+ files = os.listdir(newdir)
+ for f in files:
+ curf = os.path.join(olddir, f)
+ srcf = os.path.join(newdir, f)
+ if f in excludeFiles:
+ if os.path.exists(curf):
+ if rename:
+ cmd = 'cmp --quiet ' + srcf + ' ' + curf
+ rc = os.system(cmd);
+ if rc != 0:
+ os.rename(srcf, curf + '-new')
+ print "Keeping", f, '(new version saved as', f+'-new)'
+ else:
+ print "Adding", resf
+ os.rename(srcf, curf)
+ else:
+ if os.path.exists(curf):
+ print "Replacing", f
+ os.rename(curf, os.path.join(archive,f))
+ else:
+ print "Adding", f
+ os.rename(srcf, curf)
+
+#-----------------------------------------------------------------------------------------
+# Main program
+#-----------------------------------------------------------------------------------------
+
+if len(sys.argv) < 3 :
+ usage()
+ exit(1)
+runtime = sys.argv[1]
+tarfile = sys.argv[2]
+
+# Check if appears to be a valid (stopped) DUCC installation
+runtime = os.path.realpath(runtime)
+if not os.path.exists(os.path.join(runtime, 'resources/ducc.properties')):
+ print "ERROR - Not a valid DUCC runtime directory:", runtime
+ exit(1)
+if os.path.exists(os.path.join(runtime, 'state/ducc.pids')):
+ print "ERROR - DUCC appears to be running ... please run 'stop_ducc -a'"
+ exit(1)
+
+if not os.path.exists(tarfile):
+ print "ERROR - Missing tar file", tarfile
+ exit(1)
+
+oversion = get_oversion(runtime)
+if oversion == '1.0.0':
+ print "Sorry, migration not supported for DUCC 1.0.0 at present"
+ exit(9)
+nversion = get_nversion(os.path.basename(tarfile))
+
+#-----------------------------------------------------------------------------------------
+# Create archive directory
+#-----------------------------------------------------------------------------------------
+runtimeParent,runtimeName = os.path.split(runtime)
+now = datetime.datetime.now()
+nowstr = now.strftime('%Y%m%d-%H%M')
+archiveParent = os.path.join(runtimeParent, 'ducc_archives')
+archive = os.path.join(archiveParent, runtimeName + '_' + nowstr)
+if not os.path.exists(archive):
+ os.makedirs(archive)
+
+print " --- Updating DUCC to", nversion, " and archiving", oversion, "to",
archive
+print " NOTE: this update may be reversed by copying back all of the
archived files,"
+print " e.g. cp --recursive --remove-destination", archive+"/*", runtime
+
+#-----------------------------------------------------------------------------------------
+# Expand tarball to a peer of the runtime so can move directories
+#-----------------------------------------------------------------------------------------
+newducc = expand_tarball( tarfile, nversion, archiveParent )
+
+#-----------------------------------------------------------------------------------------
+# May need to create the new webserver directory for 2.0
+#-----------------------------------------------------------------------------------------
+weblogdir = os.path.join(runtime, 'logs/webserver')
+if not os.path.exists(weblogdir):
+ os.makedirs(weblogdir)
+
+#-----------------------------------------------------------------------------------------
+# Create a site.ducc.properties file if missing ... only for DUCC 1.1.0
+#-----------------------------------------------------------------------------------------
+siteProps = os.path.join(runtime, 'resources/site.ducc.properties')
+if not os.path.exists(siteProps):
+ if oversion != '1.1.0':
+ print "Missing site.ducc.properties - can only be created for 1.1.0"
+ exit(9)
+ currentProps = os.path.join(runtime, 'resources/ducc.properties')
+ originalProps = os.path.join(runtime, 'resources/ducc-1.1.0.properties')
+ create_110_properties(os.path.realpath(sys.argv[0]), originalProps)
+
+ if not os.path.exists(originalProps):
+ print "ERROR - Failed to create the 1.1.0 properties file from the
ending comments in this script"
+ exit(9)
+
+ # Use the new props manager - use abs fnames as otherwise are relative to
the deduced DUCC_HOME/resources
+ cmd = newducc + '/admin/ducc_props_manager --delta ' + originalProps + '
--with ' + currentProps + ' --to ' + siteProps
+ rc = os.system(cmd)
+ if rc != 0:
+ print "ERROR", rc, "Failed to create", siteProps
+ exit(9)
+ print " --- Created a file with just the site-specific properties:",
siteProps
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) all directories in the new build EXCEPT
resources & lib
+# Note that the history, logs, & state directories are not part of a build so
are left unchanged
+#-----------------------------------------------------------------------------------------
+print " --- Processing", runtimeName, "folder:"
+update_dir(runtime, newducc, archive, ['resources', 'lib'], False)
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) all files in the lib directory
+# This ensures that any site local jars remain
+#-----------------------------------------------------------------------------------------
+print " --- Processing lib folder:"
+libarchive = os.path.join(archive, 'lib')
+if not os.path.exists(libarchive):
+ os.mkdir(libarchive)
+update_dir(os.path.join(runtime, 'lib'), os.path.join(newducc, 'lib'),
libarchive, [], False)
+
+#-----------------------------------------------------------------------------------------
+# Add or replace (after archiving) some of the files in resources
+# Don't change any that are site-specific
+# (The nodes files should not be in the new build, but just in case ...)
+#-----------------------------------------------------------------------------------------
+print " --- Processing resources folder:"
+resarchive = os.path.join(archive, 'resources')
+if not os.path.exists(resarchive):
+ os.mkdir(resarchive)
+preserveFiles = [ 'ducc.classes', 'ducc.administrators', 'ducc.nodes',
'jobdriver.nodes' ]
+update_dir(os.path.join(runtime, 'resources'), os.path.join(newducc,
'resources'), resarchive, preserveFiles, True)
+
+#-----------------------------------------------------------------------------------------
+# Delete what's left of the extract (just the resources & lib folders)
+#-----------------------------------------------------------------------------------------
+shutil.rmtree(newducc)
+
+#-----------------------------------------------------------------------------------------
+# Re-build ducc_ling
+# Since it needs ducc.properties run the merge from the admin directory
+#-----------------------------------------------------------------------------------------
+print " ---"
+print " --- Rebuilding ducc_ling"
+os.chdir(runtime + '/admin')
+rc = os.system('./ducc_props_manager --merge
../resources/default.ducc.properties --with ../resources/site.ducc.properties
--to ../resources/ducc.properties')
+if (rc != 0):
+ print "ERROR - failed to create ducc.properties and to rebuild ducc_ling"
+ exit(9)
+rc = os.system('./build_duccling')
+if (rc != 0):
+ print "ERROR - failed to rebuild ducc_ling"
+ exit(9)
+
+print ""
+print " >>> Update completed!"
+print " NOTE - if your ducc_ling is privileged you should update it"
+
+#->->->->->->-> DO NOT CHANGE ANYTHING BELOW THIS MARKER <-<-<-<-<-<-<-
+#=====MARKER===== The following are the original ducc.properties shipped with
DUCC 1.1.0
+#head=<head-node>
+#jvm=<full-path-to-java-command>
+#cluster.name=Apache UIMA-DUCC
+#private.resources=${DUCC_HOME}/resources.private
+#jms.provider=activemq
+#broker.protocol=tcp
+#broker.hostname=${ducc.head}
+#broker.port=61617
+#broker.url.decoration=jms.useCompression=true
+#broker.name=localhost
+#broker.jmx.port=1100
+#broker.credentials.file=${ducc.private.resources}/ducc-broker-credentials.properties
+#broker.automanage=true
+#broker.memory.options=-Xmx1G
+#broker.configuration=conf/activemq-ducc.xml
+#broker.home=${DUCC_HOME}/apache-uima/apache-activemq
+#broker.server.url.decoration=transport.soWriteTimeout=45000
+#locale.language=en
+#locale.country=us
+#node.min.swap.threshold=0
+#admin.endpoint=ducc.admin.channel
+#admin.endpoint.type=topic
+#jmx.port=2099
+#agent.jvm.args=-Xmx500M
+#orchestrator.jvm.args=-Xmx1G
+#rm.jvm.args=-Xmx1G
+#pm.jvm.args=-Xmx1G
+#sm.jvm.args=-Xmx1G
+#db.jvm.args=-Xmx2G
+#ws.jvm.args=-Xmx2G -Djava.util.Arrays.useLegacyMergeSort=true
+#environment.propagated=USER HOME LANG
+#cli.httpclient.sotimeout=0
+#signature.required=on
+#db.configuration.class=org.apache.uima.ducc.db.config.DbComponentConfiguration
+#db.state.update.endpoint=ducc.db.state
+#db.state.update.endpoint.type=topic
+#db.state.publish.rate=15000
+#ws.configuration.class=org.apache.uima.ducc.ws.config.WebServerConfiguration
+#ws.port=42133
+#ws.port.ssl=42155
+#ws.session.minutes=60
+#ws.automatic.cancel.minutes=5
+#ws.max.history.entries=4096
+#ws.jsp.compilation.directory=/tmp/ducc/jsp
+#ws.login.enabled=false
+#ws.visualization.strip.domain=true
+#jd.configuration.class=org.apache.uima.ducc.jd.config.JobDriverConfiguration
+#jd.state.update.endpoint=ducc.jd.state
+#jd.state.update.endpoint.type=topic
+#jd.state.publish.rate=15000
+#jd.queue.prefix=ducc.jd.queue.
+#jd.queue.timeout.minutes=5
+#jd.host.class=JobDriver
+#jd.host.description=Job Driver
+#jd.host.memory.size=2GB
+#jd.host.number.of.machines=1
+#jd.host.user=System
+#jd.share.quantum=400
+#threads.limit=500
+#driver.jvm.args=-Xmx300M
+#sm.configuration.class=org.apache.uima.ducc.sm.config.ServiceManagerConfiguration
+#sm.state.update.endpoint=ducc.sm.state
+#sm.state.update.endpoint.type=topic
+#sm.default.monitor.class=org.apache.uima.ducc.cli.UimaAsPing
+#sm.instance.failure.max=5
+#sm.instance.failure.limit=${ducc.sm.instance.failure.max}
+#sm.instance.failure.window=30
+#sm.init.failure.limit=1
+#sm.meta.ping.rate=60000
+#sm.meta.ping.stability=10
+#sm.meta.ping.timeout=15000
+#sm.http.port=19989
+#sm.http.node=${ducc.head}
+#sm.default.linger=300000
+#orchestrator.configuration.class=org.apache.uima.ducc.orchestrator.config.OrchestratorConfiguration
+#orchestrator.start.type=warm
+#orchestrator.state.update.endpoint=ducc.orchestrator.state
+#orchestrator.state.update.endpoint.type=topic
+#orchestrator.state.publish.rate=10000
+#orchestrator.abbreviated.state.update.endpoint=ducc.orchestrator.abbreviated.state
+#orchestrator.abbreviated.state.update.endpoint.type=topic
+#orchestrator.abbreviated.state.publish.rate=10000
+#orchestrator.maintenance.rate=60000
+#orchestrator.http.port=19988
+#orchestrator.http.node=${ducc.head}
+#orchestrator.unmanaged.reservations.accepted=true
+#rm.configuration.class=org.apache.uima.ducc.rm.config.ResourceManagerConfiguration
+#rm.state.update.endpoint=ducc.rm.state
+#rm.state.update.endpoint.type=topic
+#rm.state.publish.rate=10000
+#rm.share.quantum=1
+#rm.scheduler=org.apache.uima.ducc.rm.scheduler.NodepoolScheduler
+#rm.class.definitions=ducc.classes
+#rm.default.memory=4
+#rm.init.stability=2
+#rm.node.stability=5
+#rm.eviction.policy=SHRINK_BY_INVESTMENT
+#rm.initialization.cap=1
+#rm.expand.by.doubling=true
+#rm.prediction=true
+#rm.prediction.fudge=120000
+#rm.fragmentation.threshold=8
+#rm.admin.endpoint=ducc.rm.admin.channel
+#rm.admin.endpoint.type=queue
+#agent.configuration.class=org.apache.uima.ducc.agent.config.AgentConfiguration
+#agent.request.endpoint=ducc.agent
+#agent.request.endpoint.type=topic
+#agent.managed.process.state.update.endpoint=ducc.managed.process.state.update
+#agent.managed.process.state.update.endpoint.type=socket
+#agent.managed.process.state.update.endpoint.params=transferExchange=true&sync=false
+#agent.node.metrics.sys.gid.max=500
+#agent.node.metrics.publish.rate=30000
+#agent.node.metrics.endpoint=ducc.node.metrics
+#agent.node.metrics.endpoint.type=topic
+#agent.node.inventory.publish.rate=10000
+#agent.node.inventory.publish.rate.skip=30
+#agent.node.inventory.endpoint=ducc.node.inventory
+#agent.node.inventory.endpoint.type=topic
+#agent.launcher.thread.pool.size=10
+#agent.launcher.use.ducc_spawn=true
+#agent.launcher.ducc_spawn_path=${DUCC_HOME}/admin/ducc_ling
+#agent.launcher.process.stop.timeout=60000
+#agent.launcher.process.init.timeout=7200000
+#agent.rogue.process.user.exclusion.filter=
+#agent.rogue.process.exclusion.filter=sshd:,-bash,-sh,/bin/sh,/bin/bash,grep,ps
+#agent.share.size.fudge.factor=5
+#agent.launcher.cgroups.enable=false
+#agent.launcher.cgroups.utils.dir=/usr/bin,/bin
+#agent.exclusion.file=${DUCC_HOME}/resources/exclusion.nodes
+#pm.configuration.class=org.apache.uima.ducc.pm.config.ProcessManagerConfiguration
+#pm.request.endpoint=ducc.pm
+#pm.request.endpoint.type=queue
+#pm.state.update.endpoint=ducc.pm.state
+#pm.state.update.endpoint.type=topic
+#pm.state.publish.rate=15000
+#uima-as.configuration.class=org.apache.uima.ducc.agent.deploy.uima.UimaAsServiceConfiguration
+#uima-as.endpoint=ducc.job.managed.service
+#uima-as.endpoint.type=socket
+#uima-as.endpoint.params=transferExchange=true&sync=false
+#uima-as.saxon.jar.path=file:${DUCC_HOME}/apache-uima/saxon/saxon8.jar
+#uima-as.dd2spring.xsl.path=${DUCC_HOME}/apache-uima/bin/dd2spring.xsl
+#flow-controller.specifier=org.apache.uima.ducc.common.uima.DuccJobProcessFC
Propchange: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/ducc_update
------------------------------------------------------------------------------
svn:executable = *
Added: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.css
URL:
http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.css?rev=1710670&view=auto
==============================================================================
--- uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.css (added)
+++ uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.css Mon Oct 26
18:31:05 2015
@@ -0,0 +1,139 @@
+
+/* start css.sty */
+.cmmi-10{font-style: italic;}
+.cmbx-12x-x-207{font-size:248%; font-weight: bold;}
+.cmbx-12x-x-207{ font-weight: bold;}
+.cmbx-12x-x-207{ font-weight: bold;}
+.cmr-12{font-size:120%;}
+.cmr-9{font-size:90%;}
+.cmbx-10{ font-weight: bold;}
+.cmbx-10{ font-weight: bold;}
+.cmbx-10{ font-weight: bold;}
+.cmtt-10{font-family: monospace;}
+.cmti-10{ font-style: italic;}
+.cmbx-12{font-size:120%; font-weight: bold;}
+.cmbx-12{ font-weight: bold;}
+.cmbx-12{ font-weight: bold;}
+.cmbxti-10{ font-weight: bold; font-style: italic;}
+.cmsl-10{font-style: oblique;}
+.cmcsc-10x-x-144{font-size:144%;}
+p.noindent { text-indent: 0em }
+td p.noindent { text-indent: 0em; margin-top:0em; }
+p.nopar { text-indent: 0em; }
+p.indent{ text-indent: 1.5em }
+@media print {div.crosslinks {visibility:hidden;}}
+a img { border-top: 0; border-left: 0; border-right: 0; }
+center { margin-top:1em; margin-bottom:1em; }
+td center { margin-top:0em; margin-bottom:0em; }
+.Canvas { position:relative; }
+img.math{vertical-align:middle;}
+li p.indent { text-indent: 0em }
+li p:first-child{ margin-top:0em; }
+li p:last-child, li div:last-child { margin-bottom:0.5em; }
+li p~ul:last-child, li p~ol:last-child{ margin-bottom:0.5em; }
+.enumerate1 {list-style-type:decimal;}
+.enumerate2 {list-style-type:lower-alpha;}
+.enumerate3 {list-style-type:lower-roman;}
+.enumerate4 {list-style-type:upper-alpha;}
+div.newtheorem { margin-bottom: 2em; margin-top: 2em;}
+.obeylines-h,.obeylines-v {white-space: nowrap; }
+div.obeylines-v p { margin-top:0; margin-bottom:0; }
+.overline{ text-decoration:overline; }
+.overline img{ border-top: 1px solid black; }
+td.displaylines {text-align:center; white-space:nowrap;}
+.centerline {text-align:center;}
+.rightline {text-align:right;}
+div.verbatim {font-family: monospace; white-space: nowrap; text-align:left;
clear:both; }
+.fbox {padding-left:3.0pt; padding-right:3.0pt; text-indent:0pt; border:solid
black 0.4pt; }
+div.fbox {display:table}
+div.center div.fbox {text-align:center; clear:both; padding-left:3.0pt;
padding-right:3.0pt; text-indent:0pt; border:solid black 0.4pt; }
+div.minipage{width:100%;}
+div.center, div.center div.center {text-align: center; margin-left:1em;
margin-right:1em;}
+div.center div {text-align: left;}
+div.flushright, div.flushright div.flushright {text-align: right;}
+div.flushright div {text-align: left;}
+div.flushleft {text-align: left;}
+.underline{ text-decoration:underline; }
+.underline img{ border-bottom: 1px solid black; margin-bottom:1pt; }
+.framebox-c, .framebox-l, .framebox-r { padding-left:3.0pt;
padding-right:3.0pt; text-indent:0pt; border:solid black 0.4pt; }
+.framebox-c {text-align:center;}
+.framebox-l {text-align:left;}
+.framebox-r {text-align:right;}
+span.thank-mark{ vertical-align: super }
+span.footnote-mark sup.textsuperscript, span.footnote-mark a
sup.textsuperscript{ font-size:80%; }
+div.tabular, div.center div.tabular {text-align: center; margin-top:0.5em;
margin-bottom:0.5em; }
+table.tabular td p{margin-top:0em;}
+table.tabular {margin-left: auto; margin-right: auto;}
+td p:first-child{ margin-top:0em; }
+td p:last-child{ margin-bottom:0em; }
+div.td00{ margin-left:0pt; margin-right:0pt; }
+div.td01{ margin-left:0pt; margin-right:5pt; }
+div.td10{ margin-left:5pt; margin-right:0pt; }
+div.td11{ margin-left:5pt; margin-right:5pt; }
+table[rules] {border-left:solid black 0.4pt; border-right:solid black 0.4pt; }
+td.td00{ padding-left:0pt; padding-right:0pt; }
+td.td01{ padding-left:0pt; padding-right:5pt; }
+td.td10{ padding-left:5pt; padding-right:0pt; }
+td.td11{ padding-left:5pt; padding-right:5pt; }
+table[rules] {border-left:solid black 0.4pt; border-right:solid black 0.4pt; }
+.hline hr, .cline hr{ height : 1px; margin:0px; }
+.tabbing-right {text-align:right;}
+span.TEX {letter-spacing: -0.125em; }
+span.TEX span.E{ position:relative;top:0.5ex;left:-0.0417em;}
+a span.TEX span.E {text-decoration: none; }
+span.LATEX span.A{ position:relative; top:-0.5ex; left:-0.4em; font-size:85%;}
+span.LATEX span.TEX{ position:relative; left: -0.4em; }
+div.float, div.figure {margin-left: auto; margin-right: auto;}
+div.float img {text-align:center;}
+div.figure img {text-align:center;}
+.marginpar {width:20%; float:right; text-align:left; margin-left:auto;
margin-top:0.5em; font-size:85%; text-decoration:underline;}
+.marginpar p{margin-top:0.4em; margin-bottom:0.4em;}
+table.equation {width:100%;}
+.equation td{text-align:center; }
+td.equation { margin-top:1em; margin-bottom:1em; }
+td.equation-label { width:5%; text-align:center; }
+td.eqnarray4 { width:5%; white-space: normal; }
+td.eqnarray2 { width:5%; }
+table.eqnarray-star, table.eqnarray {width:100%;}
+div.eqnarray{text-align:center;}
+div.array {text-align:center;}
+div.pmatrix {text-align:center;}
+table.pmatrix {width:100%;}
+span.pmatrix img{vertical-align:middle;}
+div.pmatrix {text-align:center;}
+table.pmatrix {width:100%;}
+span.bar-css {text-decoration:overline;}
+img.cdots{vertical-align:middle;}
+.partToc a, .partToc, .likepartToc a, .likepartToc {line-height: 200%;
font-weight:bold; font-size:110%;}
+.chapterToc a, .chapterToc, .likechapterToc a, .likechapterToc, .appendixToc
a, .appendixToc {line-height: 200%; font-weight:bold;}
+.index-item, .index-subitem, .index-subsubitem {display:block}
+div.caption {text-indent:-2em; margin-left:3em; margin-right:1em;
text-align:left;}
+div.caption span.id{font-weight: bold; white-space: nowrap; }
+h1.partHead{text-align: center}
+p.bibitem { text-indent: -2em; margin-left: 2em; margin-top:0.6em;
margin-bottom:0.6em; }
+p.bibitem-p { text-indent: 0em; margin-left: 2em; margin-top:0.6em;
margin-bottom:0.6em; }
+.paragraphHead, .likeparagraphHead { margin-top:2em; font-weight: bold;}
+.subparagraphHead, .likesubparagraphHead { font-weight: bold;}
+.quote {margin-bottom:0.25em; margin-top:0.25em; margin-left:1em;
margin-right:1em; text-align:justify;}
+.verse{white-space:nowrap; margin-left:2em}
+div.maketitle {text-align:center;}
+h2.titleHead{text-align:center;}
+div.maketitle{ margin-bottom: 2em; }
+div.author, div.date {text-align:center;}
+div.thanks{text-align:left; margin-left:10%; font-size:85%; font-style:italic;
}
+div.author{white-space: nowrap;}
+.quotation {margin-bottom:0.25em; margin-top:0.25em; margin-left:1em; }
+h1.partHead{text-align: center}
+ .chapterToc, .likechapterToc {margin-left:0em;}
+ .chapterToc ~ .likesectionToc, .chapterToc ~ .sectionToc, .likechapterToc ~
.likesectionToc, .likechapterToc ~ .sectionToc {margin-left:2em;}
+ .chapterToc ~ .likesectionToc ~ .likesubsectionToc, .chapterToc ~
.likesectionToc ~ .subsectionToc, .chapterToc ~ .sectionToc ~
.likesubsectionToc, .chapterToc ~ .sectionToc ~ .subsectionToc, .likechapterToc
~ .likesectionToc ~ .likesubsectionToc, .likechapterToc ~ .likesectionToc ~
.subsectionToc, .likechapterToc ~ .sectionToc ~ .likesubsectionToc,
.likechapterToc ~ .sectionToc ~ .subsectionToc {margin-left:4em;}
+.chapterToc ~ .likesectionToc ~ .likesubsectionToc ~ .likesubsubsectionToc,
.chapterToc ~ .likesectionToc ~ .likesubsectionToc ~ .subsubsectionToc,
.chapterToc ~ .likesectionToc ~ .subsectionToc ~ .likesubsubsectionToc,
.chapterToc ~ .likesectionToc ~ .subsectionToc ~ .subsubsectionToc, .chapterToc
~ .sectionToc ~ .likesubsectionToc ~ .likesubsubsectionToc, .chapterToc ~
.sectionToc ~ .likesubsectionToc ~ .subsubsectionToc, .chapterToc ~ .sectionToc
~ .subsectionToc ~ .likesubsubsectionToc, .chapterToc ~ .sectionToc ~
.subsectionToc ~ .subsubsectionToc, .likechapterToc ~ .likesectionToc ~
.likesubsectionToc ~ .likesubsubsectionToc, .likechapterToc ~ .likesectionToc ~
.likesubsectionToc ~ .subsubsectionToc, .likechapterToc ~ .likesectionToc ~
.subsectionToc ~ .likesubsubsectionToc, .likechapterToc ~ .likesectionToc ~
.subsectionToc ~ .subsubsectionToc, .likechapterToc ~ .sectionToc ~
.likesubsectionToc ~ .likesubsubsectionToc, .likechapterToc ~ .sectionToc ~
.likesubsectionToc ~ .sub
subsectionToc, .likechapterToc ~ .sectionToc ~ .subsectionToc ~
.likesubsubsectionToc .likechapterToc ~ .sectionToc ~ .subsectionToc ~
.subsubsectionToc {margin-left:6em;}
+ .likesectionToc , .sectionToc {margin-left:0em;}
+ .likesectionToc ~ .likesubsectionToc, .likesectionToc ~ .subsectionToc,
.sectionToc ~ .likesubsectionToc, .sectionToc ~ .subsectionToc
{margin-left:2em;}
+.likesectionToc ~ .likesubsectionToc ~ .likesubsubsectionToc, .likesectionToc
~ .likesubsectionToc ~ .subsubsectionToc, .likesectionToc ~ .subsectionToc ~
.likesubsubsectionToc, .likesectionToc ~ .subsectionToc ~ .subsubsectionToc,
.sectionToc ~ .likesubsectionToc ~ .likesubsubsectionToc, .sectionToc ~
.likesubsectionToc ~ .subsubsectionToc, .sectionToc ~ .subsectionToc ~
.likesubsubsectionToc, .sectionToc ~ .subsectionToc ~ .subsubsectionToc
{margin-left:4em;}
+ .likesubsectionToc, .subsectionToc {margin-left:0em;}
+ .likesubsectionToc ~ .subsubsectionToc, .subsectionToc ~ .subsubsectionToc,
{margin-left:2em;}
+.figure img.graphics {margin-left:10%;}
+dt.enumerate-enumitem{float:left; clear:left; margin-left:1em;
margin-right:1em;}
+/* end css.sty */
+
Propchange: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.pdf
URL:
http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.pdf?rev=1710670&view=auto
==============================================================================
Binary file - no diff available.
Propchange: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/duccbook.pdf
------------------------------------------------------------------------------
svn:mime-type = application/pdf
Added: uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/installation.html
URL:
http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/installation.html?rev=1710670&view=auto
==============================================================================
--- uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/installation.html
(added)
+++ uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/installation.html Mon
Oct 26 18:31:05 2015
@@ -0,0 +1,640 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd";>
+<html >
+<head><title>DUCC Installation and Verification
+Excerpt From Complete DUCC Documentation</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht
(http://www.cse.ohio-state.edu/~gurari/TeX4ht/)">
+<meta name="originator" content="TeX4ht
(http://www.cse.ohio-state.edu/~gurari/TeX4ht/)">
+<!-- html -->
+<meta name="src" content="installation.tex">
+<meta name="date" content="2015-10-15 13:07:00">
+<link rel="stylesheet" type="text/css" href="installation.css">
+</head><body
+>
+<div class="maketitle">
+
+
+
+
+
+<h2 class="titleHead">DUCC Installation and Verification<br />
+Excerpt From Complete DUCC Documentation</h2>
+<div class="author" ><span
+class="cmr-12">Written and maintained by the Apache</span>
+<br /> <span
+class="cmr-12">UIMA</span><sup class="textsuperscript"><span
+class="cmr-9">TM</span></sup><span
+class="cmr-12">Development Community</span></div>
+<br />
+<div class="date" ></div>
+</div>
+<!--l. 18--><p class="noindent" >Copyright <span
+class="cmsy-10">©</span>  2012 The Apache Software Foundation
+<!--l. 20--><p class="noindent" >Copyright <span
+class="cmsy-10">©</span>  2012 International Business Machines
Corporation
+ <!--l. 23--><p class="noindent" ><span class="paragraphHead"><a
+ id="x1-1000"></a><span
+class="cmbx-10">License and Disclaimer</span></span>
+ The ASF licenses this documentation to you under the Apache License,
Version 2.0 (the ”License”); you may not
+ use this documentation except in compliance with the License. You may
obtain a copy of the License
+ at
+ <!--l. 28--><p class="noindent" ><a
+href="http://www.apache.org/licenses/LICENSE-2.0"; class="url" ><span
+class="cmtt-10">http://www.apache.org/licenses/LICENSE-2.0</span></a>
+ <!--l. 30--><p class="noindent" >Unless required by applicable law or
agreed to in writing, this documentation and its contents are distributed under
+ the License on an ”AS IS” BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing
permissions and limitations under the
+ License.
+ <!--l. 35--><p class="noindent" ><span class="paragraphHead"><a
+ id="x1-2000"></a><span
+class="cmbx-10">Trademarks</span></span>
+ All terms mentioned in the text that are known to be trademarks or
service marks have been appropriately capitalized.
+ Use of such terms in this book should not be regarded as affecting the
validity of the the trademark or service
+ mark.
+<!--l. 47--><p class="noindent" >Publication date: October 2015
+
+
+<h3 class="sectionHead"><a
+ id="x1-3000"></a>Overview</h3>
+<!--l. 21--><p class="noindent" >DUCC is a multi-user, multi-system
distributed application. The instuctions below follow a staged
installation/verification
+methodology, roughly as follows:
+ <ul class="itemize1">
+ <li class="itemize">Single system installation.
+ </li>
+ <li class="itemize">Add new machines to DUCC control.
+ </li>
+ <li class="itemize">Enable processes to run with the credentials of
multiple submitting user. This step requires root authority on
+ one or more machines.
+ </li>
+ <li class="itemize">Enable CGroup containers. This step requires root
authority on every DUCC machine.</li></ul>
+<!--l. 36--><p class="noindent" >DUCC is distributed as a compressed tar file.
The instructions below assume installation from one of this distribution media.
+If building from source, the build creates this file in your svn trunk/target
directory. The distribution file is in the
+form
+
+
+<div class="verbatim" id="verbatim-1">
+   uima-ducc-[version]-bin.tar.gz</div>
+<!--l. 41--><p class="nopar" > where [version] is the DUCC version; for
example, <span
+class="cmti-10">uima-ducc-VERSION-bin.tar.gz </span>(where VERSION is the
current DUCC
+version). This document will refer to the distribution file as the
“<span
+class="cmmi-10"><</span>distribution.file<span
+class="cmmi-10">></span>”.
+<!--l. 45--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-4000"></a>Software Prerequisites</h3>
+<a
+ id="x1-4000doc"></a>
+<!--l. 48--><p class="noindent" >Single system installation:
+ <ul class="itemize1">
+ <li class="itemize">Reasonably current Linux. DUCC has been tested on
SLES 11.x and RHEL 6.x
+ <!--l. 53--><p class="noindent" ><span
+class="cmti-10">Note: </span>On some systems the default <span
+class="cmti-10">user limits </span>for max user processes (ulimit -u) and
nfiles (ulimit -n) are
+ defined too low for DUCC. The shell login profile for user <span
+class="cmti-10">ducc </span>should set the soft limit for max user processes
+ to be the same as the hard limit (ulimit -u ‘ulimit -Hu‘),
and the nfiles limit raised above 1024 to at least twice
+ the number of user processes running on the cluster.
+ </li>
+ <li class="itemize">Python 2.x, where ’x’ is 4 or greater.
DUCC has not been tested on Python 3.x.
+ </li>
+ <li class="itemize">Java 7. DUCC has been tested and run using IBM and
Oracle JDK 1.7.
+ </li>
+ <li class="itemize">Passwordless ssh for user running DUCC</li></ul>
+<!--l. 66--><p class="noindent" >Additional requirements for multiple system
installation:
+ <ul class="itemize1">
+ <li class="itemize">All systems must have a shared filesystem (such as
NFS or GPFS) and common user space. The $DUCC_HOME
+ directory must be located on a shared filesystem.</li></ul>
+<!--l. 73--><p class="noindent" >Additional requirements for running multiple
user processes with their own credentials.
+ <ul class="itemize1">
+ <li class="itemize">A userid <span
+class="cmti-10">ducc</span>, and group <span
+class="cmti-10">ducc</span>. User <span
+class="cmti-10">ducc </span>must the the only member of group <span
+class="cmti-10">ducc</span>.
+ </li>
+ <li class="itemize">DUCC run with user <span
+class="cmti-10">ducc </span>credentials.
+ </li>
+ <li class="itemize">Root access is required to setuid-root the DUCC
process launcher.</li></ul>
+<!--l. 81--><p class="noindent" >Additional requirements for CGroup containers:
+ <ul class="itemize1">
+ <li class="itemize">A userid <span
+class="cmti-10">ducc</span>, and group <span
+class="cmti-10">ducc</span>. User <span
+class="cmti-10">ducc </span>must the the only member of group <span
+class="cmti-10">ducc</span>.
+
+
+ </li>
+ <li class="itemize">DUCC run with user <span
+class="cmti-10">ducc </span>credentials.
+ </li>
+ <li class="itemize">libcgroup1-0.37+ on SLES and libcgroup-0.37+ on RHEL,
along with a custom /etc/cgconfig.conf</li></ul>
+<!--l. 90--><p class="noindent" >In order to build DUCC from source the
following software is also required:
+ <ul class="itemize1">
+ <li class="itemize">A Subversion client, from <a
+href="http://subversion.apache.org/packages.html"; class="url" ><span
+class="cmtt-10">http://subversion.apache.org/packages.html</span></a>. The
svn url is
+ <a
+href="https://svn.apache.org/repos/asf/uima/sandbox/uima-ducc/trunk";
class="url" ><span
+class="cmtt-10">https://svn.apache.org/repos/asf/uima/sandbox/uima-ducc/trunk</span></a>.
+ </li>
+ <li class="itemize">Apache Maven, from <a
+href="http://maven.apache.org/index.html"; class="url" ><span
+class="cmtt-10">http://maven.apache.org/index.html</span></a></li></ul>
+<!--l. 97--><p class="noindent" >The DUCC webserver server optionally supports
direct “jconsole” attach to DUCC job processes. To install this, the
+following is required:
+ <ul class="itemize1">
+ <li class="itemize">Apache Ant, any reasonably current version.</li></ul>
+<!--l. 103--><p class="noindent" >To (optionally) build the documentation, the
following is also required:
+ <ul class="itemize1">
+ <li class="itemize">Latex, including the <span
+class="cmti-10">pdflatex </span>and <span
+class="cmti-10">htlatex </span>packages. A good place to start if you
need to install it is
+ <a
+href="https://www.tug.org/texlive/"; class="url" ><span
+class="cmtt-10">https://www.tug.org/texlive/</span></a>.</li></ul>
+<!--l. 109--><p class="noindent" >More detailed one-time setup instructions
for source-level builds via subversion can be found here:
+<a
+href="http://uima.apache.org/one-time-setup.html\#svn-setup"; class="url"
><span
+class="cmtt-10">http://uima.apache.org/one-time-setup.html\#svn-setup</span></a>
+<!--l. 112--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-5000"></a>Building from Source</h3>
+<!--l. 114--><p class="noindent" >To build from source, ensure you have
Subversion and Maven installed. Extract the source from the SVN repository named
+above.
+<!--l. 117--><p class="noindent" >Then from your extract directory into the
root directory (usually current-directory/trunk), and run the command
+
+
+<div class="verbatim" id="verbatim-2">
+    mvn install</div>
+<!--l. 121--><p class="nopar" > or
+
+
+<div class="verbatim" id="verbatim-3">
+    mvn install -Pbuild-duccdocs</div>
+<!--l. 125--><p class="nopar" > if you have LaTeX insalled and wish to do the
optional build of documentation.
+<!--l. 128--><p class="noindent" >If this is your first Maven build it may
take quite a while as Maven downloads all the open-source pre-requisites. (The
+pre-requisites are stored in the Maven repository, usually your $HOME/.m2).
+<!--l. 132--><p class="noindent" >When build is complete, a tarball is placed
in your current-directory/trunk/target directory.
+<!--l. 135--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-6000"></a>Documentation</h3>
+<!--l. 137--><p class="noindent" >After installation the DUCC documentation is
found (in both PDF and HTML format) in the directory ducc_runtime/docs.
+As well, the DUCC webserver contains a link to the full documentation on each
major page. The API is documented only via
+JavaDoc, distributed in the webserver’s root directory <span
+class="cmtt-10">$DUCC</span><span
+class="cmtt-10">_HOME/webserver/root/doc/api.</span>
+<!--l. 143--><p class="noindent" >If building from source, Maven places the
documentation in
+ <ul class="itemize1">
+ <li class="itemize"><span
+class="cmtt-10">trunk/uima-ducc-duccdocs/target/site </span>(main
documentation), and
+ </li>
+ <li class="itemize"><span
+class="cmtt-10">trunk/target/site/apidocs </span>(API Javadoc)</li></ul>
+<!--l. 149--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-7000"></a>Single System Installation and Verification</h3>
+<!--l. 151--><p class="noindent" >Although any user ID can be used to run
DUCC, it is recommended to create user “ducc” to later enable use
of cgroups as
+well as running processes with the credentials of the submitting user.
+<!--l. 154--><p class="noindent" >If multiple nodes are going to be added
later, it is recommended to install the ducc runtime tree on a shared
filesystem so
+that it can be mounted on the additional nodes.
+<!--l. 157--><p class="noindent" >Verification submits a very simple UIMA
pipeline for execution under DUCC. Once this is shown to be working, one may
+proceed installing additional features.
+<!--l. 161--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-8000"></a>Minimal Hardware Requirements for Single System
Installation</h3>
+ <ul class="itemize1">
+ <li class="itemize">One Intel-based or IBM Power-based system. (More
systems may be added later.)
+ </li>
+ <li class="itemize">8GB of memory. 16GB or more is preferable for
developing and testing applications beyond the non-trivial.
+
+
+ </li>
+ <li class="itemize">1GB disk space to hold the DUCC runtime, system logs,
and job logs. More is usually needed for larger
+ installations.</li></ul>
+<!--l. 172--><p class="noindent" >Please note: DUCC is intended for scaling
out memory-intensive UIMA applications over computing clusters consisting of
+multiple nodes with large (16GB-256GB or more) memory. The minimal
requirements are for initial test and evaluation
+purposes, but will not be sufficient to run actual workloads.
+<!--l. 177--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-9000"></a>Single System Installation</h3>
+<a
+ id="x1-9000doc"></a>
+<!--l. 179--><p class="noindent" >
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-9002x1">Expand the distribution file with the
appropriate umask:
+
+
+ <div class="verbatim" id="verbatim-4">
+
(umask 022 && tar -zxf <distribution.file>)</div>
+ <!--l. 183--><p class="nopar" >
+ <!--l. 185--><p class="noindent" >This creates a directory with a name of
the form “apache-uima-ducc-[version]”.
+ <!--l. 187--><p class="noindent" >This directory contains the full DUCC
runtime which you may use “in place” but it is highly recommended
+ that you move it into a standard location on a shared filesystem; for
example, under ducc’s HOME
+ directory:
+
+
+ <div class="verbatim" id="verbatim-5">
+ mv apache-uima-ducc-[version] /home/ducc/ducc_runtime</div>
+ <!--l. 192--><p class="nopar" >
+ <!--l. 194--><p class="noindent" >We refer to this directory, regardless
of its location, as $DUCC_HOME. For simplicity, some of the examples in this
+ document assume it has been moved to /home/ducc/ducc_runtime.
+ </li>
+ <li
+ class="enumerate" id="x1-9004x2">Change directories into the admin
sub-directory of $DUCC_HOME:
+
+
+ <div class="verbatim" id="verbatim-6">
+ cd $DUCC_HOME/admin</div>
+ <!--l. 200--><p class="nopar" >
+ </li>
+ <li
+ class="enumerate" id="x1-9006x3">Run the post-installation script:
+
+
+ <div class="verbatim" id="verbatim-7">
+ ./ducc_post_install</div>
+ <!--l. 205--><p class="nopar" > If this script fails, correct any
problems it identifies and run it again.
+ <!--l. 208--><p class="noindent" >Note that <span
+class="cmti-10">ducc</span><span
+class="cmti-10">_post</span><span
+class="cmti-10">_install </span>initializes various default parameters which
may be changed later by the system
+ administrator. Therefore it usually should be run only during this first
installation step.
+ </li>
+ <li
+ class="enumerate" id="x1-9008x4">If you wish to install jconsole support
from the webserver, make sure Apache Ant is installed, and run
+
+
+ <div class="verbatim" id="verbatim-8">
+ ./sign_jconsole_jar</div>
+ <!--l. 216--><p class="nopar" > This step may be run at any time if you
wish to defer it.
+ </li></ol>
+<!--l. 221--><p class="noindent" >That’s it, DUCC is installed and ready
to run. (If errors were displayed during ducc_post_install they must be
corrected
+before continuing.)
+<!--l. 224--><p class="noindent" >The post-installation script performs these
tasks:
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-9010x1">Verifies that the correct level of Java and
Python are installed and available.
+ </li>
+ <li
+ class="enumerate" id="x1-9012x2">Creates a default nodelist,
$DUCC_HOME/resources/ducc.nodes, containing the name of the node you are
+ installing on.
+ </li>
+ <li
+ class="enumerate" id="x1-9014x3">Defines the “ducc head” node to
be to node you are installing from.
+ </li>
+ <li
+ class="enumerate" id="x1-9016x4">Sets up the default https keystore for the
webserver.
+ </li>
+ <li
+ class="enumerate" id="x1-9018x5">Installs the DUCC documentation “ducc
book” into the DUCC webserver root.
+ </li>
+ <li
+ class="enumerate" id="x1-9020x6">Builds and installs the C program,
“ducc_ling”, into the default location.
+ </li>
+ <li
+ class="enumerate" id="x1-9022x7">Ensures that the (supplied) ActiveMQ broker
is runnable.</li></ol>
+<!--l. 235--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-10000"></a>Initial System Verification</h3>
+<!--l. 237--><p class="noindent" >Here we verify the system configuration,
start DUCC, run a test Job, and then shutdown DUCC.
+<!--l. 239--><p class="noindent" >To run the verification, issue these
commands.
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-10002x1">cd $DUCC_HOME/admin
+ </li>
+ <li
+ class="enumerate" id="x1-10004x2">./check_ducc
+ <!--l. 244--><p class="noindent" >Examine the output of check_ducc. If
any errors are shown, correct the errors and rerun check_ducc until there
+ are no errors.
+ </li>
+ <li
+ class="enumerate" id="x1-10006x3">Finally, start ducc: ./start_ducc</li></ol>
+<!--l. 249--><p class="noindent" >Start_ducc will first perform a number of
consistency checks. It then starts the ActiveMQ broker, the DUCC control
+processes, and a single DUCC agent on the local node.
+<!--l. 253--><p class="noindent" >You will see some startup messages similar
to the following:
+
+
+<div class="verbatim" id="verbatim-9">
+ENV: Java is configured as: /share/jdk1.7/bin/java
+ <br />ENV: java full version "1.7.0_40-b43"
+ <br />ENV: Threading enabled: True
+ <br />MEM: memory is 15 gB
+ <br />ENV: system is Linux
+ <br />allnodes /home/ducc/ducc_runtime/resources/ducc.nodes
+ <br />Class definition file is ducc.classes
+ <br
/>OK: Class and node definitions validated.
+ <br />OK: Class configuration checked
+ <br />Starting broker on ducchead.biz.org
+ <br />Waiting for broker ..... 0
+ <br />Waiting for broker ..... 1
+ <br
/>ActiveMQ broker is found on configured host and port: ducchead.biz.org:61616
+ <br />Starting 1 agents
+ <br
/>********** Starting agents from file /home/ducc/ducc_runtime/resources/ducc.nodes
+ <br />Starting warm
+ <br />Waiting for Completion
+ <br />ducchead.biz.org Starting rm
+ <br />     PID 14198
+ <br />ducchead.biz.org Starting pm
+ <br />     PID 14223
+ <br />ducchead.biz.org Starting sm
+ <br />     PID 14248
+ <br />ducchead.biz.org Starting or
+ <br />     PID 14275
+ <br />ducchead.biz.org Starting ws
+ <br />     PID 14300
+ <br />ducchead.biz.org
+ <br />    ducc_ling OK
+ <br
/>    DUCC Agent started PID 14325
+ <br />All threads returned</div>
+<!--l. 287--><p class="nopar" >
+<!--l. 289--><p class="noindent" >Now open a browser and go to the DUCC
webserver’s url, http://<span
+class="cmmi-10"><</span>hostname<span
+class="cmmi-10">></span>:42133 where <span
+class="cmmi-10"><</span>hostname<span
+class="cmmi-10">> </span>is the name of the
+host where DUCC is started. Navigate to the Reservations page via the links in
the upper-left corner. You should see the
+DUCC JobDriver reservation in state WaitingForResources. In a few minutes this
should change to Assigned. Now jobs can
+be submitted.
+<!--l. 295--><p class="noindent" >To submit a job,
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-10008x1">$DUCC_HOME/bin/ducc_submit
–specification $DUCC_HOME/examples/simple/1.job</li></ol>
+<!--l. 300--><p class="noindent" >Open the browser in the DUCC jobs page. You
should see the job progress through a series of transitions: Waiting For
+Driver, Waiting For Services, Waiting For Resources, Initializing, and
finally, Running. You’ll see the number of work items
+submitted (15) and the number of work items completed grow from 0 to 15.
Finally, the job will move into Completing and
+then Completed..
+<!--l. 306--><p class="noindent" >Since this example does not specify a log
directory DUCC will create a log directory in your HOME directory
+under
+
+
+<div class="verbatim" id="verbatim-10">
+$HOME/ducc/logs/job-id</div>
+<!--l. 309--><p class="nopar" >
+<!--l. 311--><p class="noindent" >In this directory, you will find a log for
the sample job’s JobDriver (JD), JobProcess (JP), and a number of other
files
+relating to the job.
+<!--l. 314--><p class="noindent" >This is a good time to explore the DUCC web
pages. Notice that the job id is a link to a set of pages with details about the
+execution of the job.
+<!--l. 317--><p class="noindent" >Notice also, in the upper-right corner is a
link to the full DUCC documentation, the “DuccBook”.
+<!--l. 319--><p class="noindent" >Finally, stop DUCC:
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-10010x1">cd $DUCC_HOME/admin
+ </li>
+ <li
+ class="enumerate" id="x1-10012x2">./stop_ducc -a</li></ol>
+<!--l. 326--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-11000"></a>Add additional nodes to the DUCC cluster</h3>
+<!--l. 327--><p class="noindent" >Additional nodes must meet all <span
+class="cmti-10">prerequisites </span>(listed above).
+<!--l. 334--><p class="noindent" >$DUCC_HOME must be on a shared filesystem
and mounted at the same location on all DUCC nodes.
+<!--l. 337--><p class="noindent" >If user’s home directories are on
local filesystems the location for user logfiles should be specified to be on a
shared
+filesystem.
+<!--l. 340--><p class="noindent" >Addional nodes are normally added to a
worker node group. Note that the DUCC head node does not have to be a worker
+node. In addition, the webserver node can be separate from the DUCC head node
(see webserver configuration options in
+ducc.properties).
+<!--l. 345--><p class="noindent" >For worker nodes DUCC needs to know what
node group each machine belongs to, and what nodes need an Agent process to
+be started on.
+<!--l. 348--><p class="noindent" >The configuration shipped with DUCC have all
nodes in the same ”default” node pool. Worker nodes are listed in
the
+file
+
+
+<div class="verbatim" id="verbatim-11">
+$DUCC_HOME/resources/ducc.nodes.</div>
+<!--l. 352--><p class="nopar" >
+<!--l. 354--><p class="noindent" >During initial installation, this file was
initialized with the node DUCC is installed on. Additional nodes may be added
to the
+file using a text editor to increase the size of the DUCC cluster.
+<!--l. 359--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-12000"></a>Ducc_ling Configuration - Running with credentials of
submitting user</h3>
+<a
+ id="x1-12000doc"></a>
+<!--l. 362--><p class="noindent" >DUCC launches user processes through
ducc_ling, a small native C application. By default the resultant process runs
with
+the credentials of the user ID of the DUCC application. It is possible for
multiple users to submit work to DUCC in this
+configuration, but it requires that the user ID running DUCC has write access
to all directories to which the user process
+outputs data. By configuring the ducc user ID and ducc_ling correctly, work
submitted by all users will run with their own
+credentials.
+<!--l. 370--><p class="noindent" >Before proceeding with this step, please
note:
+ <ul class="itemize1">
+ <li class="itemize">The sequence operations consisting of <span
+class="cmti-10">chown </span>and <span
+class="cmti-10">chmod </span>MUST be performed in the exact order given below.
+ If the <span
+class="cmti-10">chmod </span>operation is performed before the <span
+class="cmti-10">chown </span>operation, Linux will regress the permissions
granted by
+ <span
+class="cmti-10">chmod </span>and ducc_ling will be incorrectly
installed.</li></ul>
+<!--l. 378--><p class="noindent" >ducc_ling is designed to be a setuid-root
program whose function is to run user processes with the identity of the
submitting
+user. This must be installed correctly; incorrect installation can prevent
jobs from running as their submitters, and in the
+worse case, can introduce security problems into the system.
+<!--l. 382--><p class="noindent" >ducc_ling can either be installed on a local
disk on every system in the DUCC cluster, or on a shared-filesystem that does
not
+suppress setuid-root permissions on client nodes. The path to ducc_ling must
be the same on each DUCC node. The default
+path configuration is $DUCC_HOME/admin/$<span
+class="cmsy-10">{</span>os.arch<span
+class="cmsy-10">}</span>/ in order to handle clusters with mixed OS platforms.
+$<span
+class="cmsy-10">{</span>os.arch<span
+class="cmsy-10">} </span>is the architecture specific value of the Java system
property with that name; examples are amd64 and
+ppc64.
+<!--l. 390--><p class="noindent" >The steps are: build ducc_ling for each node
architecture to be added to the cluster, copy ducc_ling to the
+desired location, and then configure ducc_ling to give user ducc the ability
to spawn a process as a different
+user.
+<!--l. 394--><p class="noindent" >In the example below ducc_ling is left under
$DUCC_HOME, where it is built.
+<!--l. 396--><p class="noindent" >As user <span
+class="cmti-10">ducc</span>, build ducc_ling for necessary architectures (this
is done automatically for the DUCC head machine by the
+ducc_post_install script). For each unique OS platform:
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-12002x1">cd $DUCC_HOME/admin
+ </li>
+ <li
+ class="enumerate" id="x1-12004x2">./build_duccling</li></ol>
+<!--l. 404--><p class="noindent" >Then, as user <span
+class="cmti-10">root </span>on the shared filesystem, <span
+class="cmti-10">cd </span><span
+class="cmsl-10">$</span><span
+class="cmti-10">DUCC</span><span
+class="cmti-10">_HOME/admin</span>, and for each unique OS architecture:
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-12006x1">chown ducc.ducc $<span
+class="cmsy-10">{</span>os.arch<span
+class="cmsy-10">}</span><br
+class="newline" />(set directory ownership to be user ducc, group ducc)
+
+
+ </li>
+ <li
+ class="enumerate" id="x1-12008x2">chmod 700 $<span
+class="cmsy-10">{</span>os.arch<span
+class="cmsy-10">}</span><br
+class="newline" />(only user ducc can read contents of directory)
+ </li>
+ <li
+ class="enumerate" id="x1-12010x3">chown root.ducc $<span
+class="cmsy-10">{</span>os.arch<span
+class="cmsy-10">}</span>/ducc_ling <br
+class="newline" />(make root owner of ducc_ling, and let users in group ducc
access it)
+ </li>
+ <li
+ class="enumerate" id="x1-12012x4">chmod 4750 $<span
+class="cmsy-10">{</span>os.arch<span
+class="cmsy-10">}</span>/ducc_ling <br
+class="newline" />(ducc_ling runs as user root when started by users in group
ducc)</li></ol>
+<!--l. 416--><p class="noindent" >If these steps are correctly performed, ONLY
user <span
+class="cmti-10">ducc </span>may use the ducc_ling program in a privileged way.
ducc_ling
+contains checks to prevent even user <span
+class="cmti-10">root </span>from using it for privileged operations.
+<!--l. 420--><p class="noindent" >If a different location is chosen for
ducc_ling the new path needs to be specified for
ducc.agent.launcher.ducc_spawn_path in
+$DUCC_HOME/resources/site.ducc.properties. See more info at see <span
+class="cmti-10">properties merging </span>in the duccbook.
+<!--l. 430--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-13000"></a>CGroups Installation and Configuration</h3>
+<!--l. 432--><p class="noindent" >
+ <dl class="description"><dt class="description">
+<span
+class="cmbx-10">Note:</span> </dt><dd
+class="description">A key feature of DUCC is to run user processes in CGroups
in order to guarantee each process always has
+ the amount of RAM requested. RAM allocated to the managed process (and
any child processes) that exceed
+ requested DUCC memory size will be forced into swap space. Without
CGroups a process that exceeds its
+ requested memory size by N% is killed (default N=5 in ducc.properties),
and memory use by child processes is
+ ignored.
+ <!--l. 439--><p class="noindent" >DUCC’s CGroup configuration also
allocates CPU resources to managed processes based on relative memory
+ size. A process with 50% of a machine’s RAM will be guaranteed at
least 50% of the machine’s CPU resources
+ as well.</dd></dl>
+<!--l. 444--><p class="noindent" >The steps in this task must be done as user
root and user ducc.
+<!--l. 446--><p class="noindent" >To install and configure CGroups for DUCC:
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x1-13002x1">Install the appropriate libcgroup package
at level 0.37 or above (see <span
+class="cmti-10">Installation Prerequisites</span>).
+ </li>
+ <li
+ class="enumerate" id="x1-13004x2">Configure /etc/cgconfig.conf as follows:
+
+
+ <div class="verbatim" id="verbatim-12">
+    # Mount cgroups
+  <br />   mount {
+  <br
/>      memory = /cgroup;
+  <br
/>      cpu = /cgroup;
+  <br />   }
+  <br
/>   # Define cgroup ducc and setup permissions
+  <br />   group ducc {
+  <br />    perm {
+  <br
/>        task {
+  <br
/>           uid = ducc;
+  <br
/>        }
+  <br
/>        admin {
+  <br
/>           uid = ducc;
+  <br
/>        }
+  <br />    }
+  <br />    memory {}
+  <br />    cpu{}
+  <br />   }</div>
+ <!--l. 475--><p class="nopar" >
+ </li>
+ <li
+ class="enumerate" id="x1-13006x3">Start the cgconfig service:
+
+
+ <div class="verbatim" id="verbatim-13">
+    service cgconfig start</div>
+ <!--l. 479--><p class="nopar" >
+ </li>
+ <li
+ class="enumerate" id="x1-13008x4">Verify cgconfig service is running by the
existence of directory:
+
+
+ <div class="verbatim" id="verbatim-14">
+    /cgroups/ducc</div>
+ <!--l. 484--><p class="nopar" >
+ </li>
+ <li
+ class="enumerate" id="x1-13010x5">Configure the cgconfig service to start on
reboot:
+
+
+ <div class="verbatim" id="verbatim-15">
+    chkconfig cgconfig on</div>
+ <!--l. 489--><p class="nopar" ></li></ol>
+<!--l. 492--><p class="noindent" ><span
+class="cmti-10">Note: </span>if CGroups is not installed on a machine the DUCC
Agent will detect this and not attempt to use the feature. CGroups
+can also be disabled for all machines (see <span
+class="cmti-10">ducc.agent.launcher.cgroups.enable </span>in ducc.properties,
described in the Duccbook.)
+or it can be disabled for individual machines (see <span
+class="cmti-10">ducc.agent.exclusion.file </span>in ducc.properties, described
in the
+Duccbook.)
+<!--l. 509--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-14000"></a>Full DUCC Verification</h3>
+<!--l. 511--><p class="noindent" >This is identical to initial verification,
with the one difference that the job “1.job” should be submitted as
any user other
+than ducc. Watch the webserver and check that the job executes under the
correct identity. Once this completes, DUCC is
+installed and verified.
+<!--l. 515--><p class="noindent" >
+<h3 class="sectionHead"><a
+ id="x1-15000"></a>Enable DUCC webserver login</h3>
+<!--l. 517--><p class="noindent" >This step is optional. As shipped, the
webserver is disabled for logins. This can be seen by hovering over the Login
text
+located in the upper right of most webserver pages:
+
+
+<div class="verbatim" id="verbatim-16">
+System is configured to disallow logins</div>
+<!--l. 522--><p class="nopar" >
+<!--l. 524--><p class="noindent" >To enable logins, a Java-based authenticator
must be plugged-in and the login feature must be enabled in the ducc.properties
+file by the DUCC administrator. Also, ducc_ling should be properly deployed
(see Ducc_ling Installation section
+above).
+<!--l. 529--><p class="noindent" >A beta version of a Linux-based
authentication plug-in is shipped with DUCC. It can be found in the source
+tree:
+
+
+<div class="verbatim" id="verbatim-17">
+org.apache.uima.ducc.ws.authentication.LinuxAuthenticationManager</div>
+<!--l. 533--><p class="nopar" >
+<!--l. 535--><p class="noindent" >The Linux-based authentication plug-in will
attempt to validate webserver login requests by appealing to the host OS. The
+user who wishes to login provides a userid and password to the webserver via
https, which in-turn are handed-off to the OS
+for a success/failure reply.
+<!--l. 540--><p class="noindent" >To have the webserver employ the beta
Linux-based authentication plug-in, the DUCC administrator should perform the
+following as user ducc:
+
+
+<div class="verbatim" id="verbatim-18">
+1. edit ducc.properties
+ <br
/>2. locate: ducc.ws.login.enabled = false
+ <br
/>3. modify: ducc.ws.login.enabled = true
+ <br />4. save</div>
+<!--l. 547--><p class="nopar" >
+<!--l. 549--><p class="noindent" >Note: The beta Linux-based authentication
plug-in has limited testing. In particular, it was tested using:
+
+
+<div class="verbatim" id="verbatim-19">
+Red Hat Enterprise Linux Workstation release 6.4 (Santiago)</div>
+<!--l. 553--><p class="nopar" >
+<!--l. 555--><p class="noindent" >Alternatively, you can provide your own
authentication plug-in. To do so:
+
+
+<div class="verbatim" id="verbatim-20">
+1. author a Java class that implements
+ <br
/>   org.apache.uima.ducc.common.authentication.IAuthenticationManager
+ <br
/>2. create a jar file comprising your authentication class
+ <br
/>3. put the jar file in a location accessible by the DUCC webserver, such as
+ <br />    $DUCC_HOME/lib/authentication
+ <br
/>4. put any authentication dependency jar files there as well
+ <br />5. edit ducc.properties
+ <br />6. add the following:
+ <br
/>   ducc.local.jars = authentication/*
+ <br
/>   ducc.authentication.implementer=<your.authenticator.class.Name>
+ <br
/>7. locate: ducc.ws.login.enabled = false
+ <br
/>8. modify: ducc.ws.login.enabled = true
+ <br />9. save</div>
+<!--l. 570--><p class="nopar" >
+
+</body></html>
+
+
+
+
Propchange:
uima/site/trunk/uima-website/docs/d/uima-ducc-2.0.1/installation.html
------------------------------------------------------------------------------
svn:eol-style = native
