Author: degenaro
Date: Sat Sep 19 14:19:34 2015
New Revision: 1704040
URL: http://svn.apache.org/viewvc?rev=1704040&view=rev
Log:
UIMA-4569 DUCC OR+WS expunge "shares" (fix TestSuite for JD Scheduler)
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties
(with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml
(with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties
(with props)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java?rev=1704040&r1=1704039&r2=1704040&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
Sat Sep 19 14:19:34 2015
@@ -55,24 +55,16 @@ public class TestSuite {
private static DuccLogger logger = null;
@Before
- public void testBefore() {
- }
-
- @After
- public void testAfter() {
- }
-
- @Test
- public void test() {
- }
-
- //@Before
public void before() {
ducc_home();
ducc_logger();
ducc_config();
}
+ @After
+ public void after() {
+ }
+
private void ducc_logger() {
logger = new DuccLogger(TestSuite.class);
DuccService.setDuccLogger(logger);
@@ -115,10 +107,6 @@ public class TestSuite {
String key = "DUCC_HOME";
System.setProperty(key, value);
}
-
- //@After
- public void after() {
- }
private int seed = 0;
private Random random = new Random(seed);
@@ -169,7 +157,7 @@ public class TestSuite {
}
}
- //@Test
+ @Test
public void testJdReservation() {
try {
IDuccWorkReservation dwr = new DuccWorkReservation();
@@ -191,7 +179,7 @@ public class TestSuite {
}
}
- //@Test
+ @Test
public void testJdSchedulerStatic() {
try {
JdHostProperties jdHostProperties = new
JdHostProperties();
@@ -246,7 +234,7 @@ public class TestSuite {
}
}
- //@Test
+ @Test
public void testJdSchedulerDynamic() {
try {
DuccId jobId = new DuccId(999);
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties?rev=1704040&view=auto
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties
Sat Sep 19 14:19:34 2015
@@ -0,0 +1,583 @@
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: <head-node>
+ducc.head = test
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value:
<full-path-to-java-command>
+ducc.jvm = /usr/bin/java
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: Apache UIMA-DUCC
+ducc.cluster.name = Test DUCC
+
+# Specify location of private resources directory. UIMA-3892
+ducc.private.resources = ${DUCC_HOME}/resources.private
+
+# Name any site-local jars
+#ducc.local.jars =
+# Declare which components to load into the jvm on process startup.
+ducc.jms.provider = activemq
+
+# ---------------------------------------------------
+# ---------------------------------------------------
+# Define the ActivemQ broker configuration.
+# ducc.broker.protocol - Ddefine broker protocol. Should typically be
+# tcp. *** Dont specify : or // ***
+# ducc.broker.hostname - The name of the host where the broker is running.
+# ducc.broker.port - The broker's connection port
+# ducc.broker.url.decoration - The broker *client* url decoration.
+# recoveryInterval=30000 - Specifies an interval between recovery
attempts,
+# i.e when a connection is being refreshed,
+# in milliseconds
+# jms.useCompression=true - Compress message body
+# ducc.broker.name - the broker name - must match the actual broker name
+# in the broker config. THIS IS NOT THE BROKER
HOSTNAME
+# WHICH IS CONFIGURED IN ducc.broker.hostname.
+# ducc.broker.jmx.port - the Broker's jmx port
+#
+ducc.broker.protocol = tcp
+
+ducc.broker.hostname = ${ducc.head}
+
+ducc.broker.port = 61617
+
+# Broker decorations:
+# useCompression: true, compress JMS cargo
+ducc.broker.url.decoration = jms.useCompression=true
+
+ducc.broker.name = localhost
+
+ducc.broker.jmx.port = 1100
+
+#
+# ActiveMQ credentials file used to authenticate DUCC daemons with the broker.
+#
+ducc.broker.credentials.file =
${ducc.private.resources}/ducc-broker-credentials.properties
+
+# ActiveMQ Auto-management configuration
+# docc.broker.automanage - if true, DUCC will start and stop the broker
as needed.
+# Otherwise, the installation is responsible for
+# manageing ActiveMQ
+# ducc.broker.memor.options - If automanaged, the memory configuration
passed to the JVM
+# ducc.broker.configuration - If automanaged, the location of the ActiveMQ
broker configuration
+# ducc.broker.home - If automanaged, the location of the ActiveMQ
installation
+# ducc.broker.server.url.decoration - If automanaged, the broker URL
decoration
+#
+ducc.broker.automanage = true
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx1G
+ducc.broker.memory.options = -Xmx500M
+
+ducc.broker.configuration = conf/activemq-ducc.xml
+
+ducc.broker.home = ${DUCC_HOME}/apache-uima/apache-activemq
+
+# transport.soWriteTimeout=45000, 45sec timeout on write operations to the
socket.
+ducc.broker.server.url.decoration = transport.soWriteTimeout=45000
+
+# To enable tracing of RM messages arriving in OR and NodeMetrics arriving in
WS.
+#ducc.transport.trace = orchestrator:RmStateDuccEvent
webserver:NodeMetricsUpdateDuccEvent
+# By default, use Linux OS authentication (via PAM)
+#ducc.authentication.implementer=org.apache.uima.ducc.ws.authentication.LinuxAuthenticationManager
+# Specify users allowed to login (default is all users can login)
+#ducc.authentication.users.include = user1 user2
+# Specify users not allowed to login (default is all users can login; if user
is in both include and exclude lists, then exclude is enforced)
+#ducc.authentication.users.exclude = user1 user3
+# Specify groups allowed to login (default is all groups can login)
+#ducc.authentication.groups.include = groupA groupB
+# Specify groups not allowed to login (default is all groups can login; if
group is in both include and exclude lists, then exclude is enforced)
+#ducc.authentication.groups.exclude = groupA groupC
+# Access UIMA message catalogs
+ducc.locale.language = en
+
+ducc.locale.country = us
+
+# Specify a minimum amount of free swap space available on a node.
+# If an agent detects free swap space dipping below the value defined
+# below, it will find the fattest (in terms of memory) process in its
+# inventory and kill it. The value of the parameter below is expressed
+# in MB.
+# Initially disabled by setting the threshold at 0.
+# Set this to 1GB equiv
+ducc.node.min.swap.threshold = 0
+
+# administrative endpoint for all ducc components
+ducc.admin.endpoint = ducc.admin.channel
+
+# endpoint type choices[topic]
+ducc.admin.endpoint.type = topic
+
+# jmx port number for Ducc process. Each Ducc process will attempt
+# to use this port for its JMX Connector. If the port is not available
+# port+1 will be used until an available port is found.
+ducc.jmx.port = 2099
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx500M
+ducc.agent.jvm.args = -Xmx250M
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx1G
+ducc.orchestrator.jvm.args = -Xmx500M
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx1G
+ducc.rm.jvm.args = -Xmx500M
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx1G
+ducc.pm.jvm.args = -Xmx500M
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx1G
+ducc.sm.jvm.args = -Xmx500M
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: -Xmx2G
-Djava.util.Arrays.useLegacyMergeSort=true
+ducc.ws.jvm.args = -Xmx1G -Djava.util.Arrays.useLegacyMergeSort=true
+
+# ========== CLI Configuration block ==========
+# These environment values are included on job/service/AP submissions
+ducc.environment.propagated = USER HOME LANG DUCC_SERVICE_INSTANCE
+
+# No timeout on CLI requests
+ducc.cli.httpclient.sotimeout = 0
+
+#------------------------------------------------------------------------------
+# off, // CLI submit and cancel signature enforcement disabled
+# on, // CLI submit and cancel signature enforcement enabled (default)
+ducc.signature.required = on
+
+#------------------------------------------------------------------------------
+# ========== Web Server Configuration block ==========
+ducc.ws.configuration.class =
org.apache.uima.ducc.ws.config.WebServerConfiguration
+
+# Optionally configure the webserver to run on a non-head node
+# ducc.ws.node = my.node.com
+# Optionally configure the webserver IP address
+# ducc.ws.ipaddress = <fill in an IP address>
+# Optionally configure the webserver IP port for HTTP requests, default is
42133
+ducc.ws.port = 42133
+
+# Optionally configure the webserver IP port for HTTPS requests, default is
42155
+ducc.ws.port.ssl = 42155
+
+# Optionally configure the webserver ssl pw for HTTPS requests, default is
quackquack
+# See private/ducc.private.properties
+# Optionally configure the webserver login session timeout, in minutes,
default is 60
+ducc.ws.session.minutes = 60
+
+# Optionally configure the webserver job automatic cancel timeout, default is
10. To disable feature specify 0.
+# Employed when user specifies --wait_for_completion flag on job submission,
in which case the job monitor
+# program must visit
http://<host>:<port>/ducc-servlet/proxy-job-status?id=<job-id> within this
expiry time
+# else the job will be automatically canceled (unless feature disabled) by
the webserver acting as the
+# administrator ducc (which must be specified in the ducc.administrators
file).
+ducc.ws.automatic.cancel.minutes = 5
+
+# Optionally configure the webserver max history entries for each of
Jobs/Reservations/Services
+ducc.ws.max.history.entries = 4096
+
+# Specify the ducc jsp comp dir
+ducc.ws.jsp.compilation.directory = /tmp/ducc/jsp
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: false
+ducc.ws.login.enabled = true
+
+# For node visualization - if true,strip domain names from labels for cleaner
visuals
+ducc.ws.visualization.strip.domain = true
+
+# Optionally configure the webserver request log, default is 0 (meaning no
request logging)
+# Logs are written to DUCC_HOME/logs/webserver
+ducc.ws.requestLog.RetainDays = 30
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value:
org.apache.uima.ducc.transport.configuration.jd.JobDriverConfiguration
+ducc.jd.configuration.class =
org.apache.uima.ducc.transport.configuration.jd.JobDriverConfiguration
+
+ducc.jd.state.update.endpoint = ducc.jd.state
+
+# endpoint type choices[topic]
+ducc.jd.state.update.endpoint.type = topic
+
+ducc.jd.state.publish.rate = 15000
+
+# This is a human-readable string used to form queue names for the JMS queues
used to pass
+# CASs from the Job Driver to the Job Processes. The completed queue named
comprises the prefix
+# concatenated with the DUCC assigned Job number.
+ducc.jd.queue.prefix = ducc.jd.queue.
+
+# After dispatching a work item to UIMA-AS client for processing, the number
of minutes that the Job Driver will
+# wait for two callbacks (queued and assigned) before considering the work
item lost. The elapsed time for the
+# callbacks is normally sub-second. Intermittent network problems may cause
unusual spikes.
+# If not specified, default value is 5 minutes.
+ducc.jd.queue.timeout.minutes = 5
+
+# The following 5 values comprise the specification used by the DUCC
Orchestrator daemon to request an
+# allocation from the DUCC Resource Manager for Job Driver use. The values
given below are the defaults.
+ducc.jd.host.class = JobDriver
+
+ducc.jd.host.description = Job Driver
+
+ducc.jd.host.memory.size = 2GB
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: 1
+ducc.jd.host.number.of.machines = 2
+
+ducc.jd.host.user = System
+
+# For a newly started Job, the number of JP UIMA initialization failures
+# allowed until at least one JP succeeds - otherwise, the Job
self-destructs.
+# Default is 1.
+ducc.jd.startup.initialization.error.limit = 1
+
+# The next 4 values are related - each JD is assigned a piece of the Job
Driver host memory which,
+# along with the size of the CR's type system, limits the number of active
work-item CASes in a job.
+# To avoid swapping the max heap size should also be restricted.
+# Memory size in MB allocated for each JD (default 300)
+ducc.jd.share.quantum = 400
+
+# The number of "slices" of size "jd.share.quantum" kept in reserve.
+# The Orchestrator makes Reservation requests to RM to get Reservations
+# (Job Driver hosts) each of which is then subdivided into "slices", one
+# per JD. This number specifies the number of unused "slices" that should
+# be kept on-hand in anticipation of newly submitted jobs (default 2)
+ducc.jd.share.quantum.reserve.count = 3
+
+# Max number of work-item CASes for each job (default is "unlimited")
+ducc.threads.limit = 500
+
+# Extra JVM args to be appended to any user-supplied "driver_jvm_args"
+# Dynamic: changes to this property immediately affect new submitted work
+# Flag: -DUimaAsCasTracking will add to logs:
+# UIMA-AS client & service trigger internal tracing including RefId,
+# UIMA-AS client onBeforeMessageSendHandler callbacks including RefId,
and
+# UIMA-AS client onBeforeProcessCAS callbacks including RefId
+# Flag: -DUimaAsClientTracking will add to jd.out.log:
+# core pool size changes,
+# UIMA-AS client sendAndReceive invocations,
+# UIMA-AS client onBeforeMessageSendHandler callbacks, and
+# UIMA-AS client onBeforeProcessCAS callbacks
+# Note: should contain an Xmx a little below the "ducc.jd.share.quantum" value
+# the Xmx entry is ignored if the user has speccified one
+ducc.driver.jvm.args = -Xmx300M
+
+# Max number of threads in Jetty thread pool servicing incoming
+# HTTP requests
+# ducc.driver.jetty.max.threads = 100
+# Max idle time for jetty threads (in millis). When a thread exceeds
+# its idle time it will be terminated.
+# ducc.driver.jetty.thread.idletime = 60000
+# Extra JVM args to be appended to any user-supplied "process_jvm_args"
+# Dynamic: changes to this property immediately affect new submitted work
+# ducc.process.jvm.args=-XX:+HeapDumpOnOutOfMemoryError
+# ========== Job Driver Configuration block ==========
+# ========== Service Manager Configuration block ==========
+# Class used to implement the SM - internals only, don't change.
+ducc.sm.configuration.class =
org.apache.uima.ducc.sm.config.ServiceManagerConfiguration
+
+# Name of the AMQ topic used to communicate wiht SM - internals, don't change
+ducc.sm.state.update.endpoint = ducc.sm.state
+
+# AMQ protocol to use - internals, don't change
+ducc.sm.state.update.endpoint.type = topic
+
+# Default pinger
+ducc.sm.default.monitor.class = org.apache.uima.ducc.cli.UimaAsPing
+
+# max consecutive init failures or ping failures for ping-only services before
we stop starting things
+ducc.sm.instance.failure.max = 5
+
+# backward compatibility
+ducc.sm.instance.failure.limit = ${ducc.sm.instance.failure.max}
+
+# failure window size in MINUTES.
+ducc.sm.instance.failure.window = 30
+
+# max consecutive instance init failures before we stop trying to start things
+ducc.sm.init.failure.limit = 1
+
+# How often services are monitored to insure they're responding, in
milliseconds
+ducc.sm.meta.ping.rate = 60000
+
+# How many consecutive missed pings are required to mark a service unresponive
+ducc.sm.meta.ping.stability = 10
+
+# How long to wait for a ping to respond, in milliseconds
+ducc.sm.meta.ping.timeout = 15000
+
+# Port the CLI uses to contact SM
+ducc.sm.http.port = 19989
+
+# Node where SM resides
+ducc.sm.http.node = ${ducc.head}
+
+# How long to keep a reference-started service alive after the last
referencing job exits
+ducc.sm.default.linger = 300000
+
+# === END == Service Manager Configuration block ==========
+# ========== Orchestrator Configuration block ==========
+ducc.orchestrator.configuration.class =
org.apache.uima.ducc.orchestrator.config.OrchestratorConfiguration
+
+#------------------------------------------------------------------------------
+# cold, // Recover: All is lost
JD host: employ new
+# warm, // Recover: Unmanaged Reservations only (default) JD
host: employ new
+ducc.orchestrator.start.type = warm
+
+#------------------------------------------------------------------------------
+ducc.orchestrator.state.update.endpoint = ducc.orchestrator.state
+
+# endpoint type choices[topic]
+ducc.orchestrator.state.update.endpoint.type = topic
+
+ducc.orchestrator.state.publish.rate = 10000
+
+#------------------------------------------------------------------------------
+# How long between between maintenance cycles (in milliseconds)
+# Each maintenance cycle the orchestrator removes obsolete JD queues (MqReaper)
+# and performs health related activities (HealthMonitor) including: capping JPs
+# for a Jobs that have too many initialization failures and terminating Jobs
whose
+# JDs have failed.
+ducc.orchestrator.maintenance.rate = 60000
+
+#------------------------------------------------------------------------------
+# orchestrator's jetty http port
+ducc.orchestrator.http.port = 19988
+
+# Node where OR is running. This is needed by CLI to compose a URL to access
OR jetty server
+ducc.orchestrator.http.node = ${ducc.head}
+
+#------------------------------------------------------------------------------
+# Specify if reserving an entire machine is allowed
+# This could be a bad idea when a cluster has very few, very large machines.
+ducc.orchestrator.unmanaged.reservations.accepted = true
+
+# ========== Orchestrator Configuration block ==========
+# Resource Manager Configuration block
+ducc.rm.configuration.class =
org.apache.uima.ducc.rm.config.ResourceManagerConfiguration
+
+# Queue name for broadcast RM state
+ducc.rm.state.update.endpoint = ducc.rm.state
+
+# If enabled, RM tries to start as soon as it recoveres state from an OR
publication,
+# instread of waiting for init.stability for nodes to check in.
+ducc.rm.state.update.endpoint.type = topic
+
+# the frequency, relative to or publicatations, at which RM runs a schedule
+ducc.rm.state.publish.ratio = 1
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: 1
+ducc.rm.share.quantum = 15
+
+# Implementation class for actual scheduling algorithm
+ducc.rm.scheduler = org.apache.uima.ducc.rm.scheduler.NodepoolScheduler
+
+# File defining the scheduler classes - found in DUCC_HOME/resources
+ducc.rm.class.definitions = ducc.classes
+
+# File defining the user registry. Optional, need not exist. Found in
DUCC_HOME/resources.
+ducc.rm.user.registry = ducc.users
+
+# default memory, in GB, if not specified
+ducc.rm.default.memory = 4
+
+# number of node metrics heartbeats to wait for before rm starts up
+ducc.rm.init.stability = 2
+
+# number of missed node metrics updates to consider node down
+ducc.rm.node.stability = 5
+
+# which policy to use when shrinking/evicting shares - alternatively,
SHRINK_BY_MACHINE
+ducc.rm.eviction.policy = SHRINK_BY_INVESTMENT
+
+# max nodes to initially allocate until init is complete
+ducc.rm.initialization.cap = 1
+
+# When true, jobs expand not all at once after init, but a bit slower,
doubling each epoch
+# until max fair-share is set. If false, jobs increase immediately to their
fair share,
+# at the cost of mass evictions.
+ducc.rm.expand.by.doubling = true
+
+# Predict when a job will end and avoid expanding if not needed
+ducc.rm.prediction = true
+
+# Add this fudge factor (milliseconds) to the expansion target when using
prediction
+ducc.rm.prediction.fudge = 120000
+
+# What is minimum number of processes for a job before we do defrag? If a job
has less than this amount
+# RM may attempt defragmentation to bring the processes up to this value.
+ducc.rm.fragmentation.threshold = 8
+
+# Endpoint for RM admin CLI
+ducc.rm.admin.endpoint = ducc.rm.admin.channel
+
+ducc.rm.admin.endpoint.type = queue
+
+# Agent Configuration block
+ducc.agent.configuration.class =
org.apache.uima.ducc.agent.config.AgentConfiguration
+
+ducc.agent.request.endpoint = ducc.agent
+
+# endpoint type choices[topic]
+ducc.agent.request.endpoint.type = topic
+
+# Endpoint where uima as service wrapper reports status
+ducc.agent.managed.process.state.update.endpoint =
ducc.managed.process.state.update
+
+# endpoint type choices[socket]
+ducc.agent.managed.process.state.update.endpoint.type = socket
+
+# endpoint parameters which are transport specific. For socket
+# transport params are:
+# - transferExchange=true - include Camel Exchange wrapper in a message
+# - synch=false - use socket transport for one-way messaging (no replies
neeeded)
+ducc.agent.managed.process.state.update.endpoint.params =
transferExchange=true&sync=false
+
+# max UID reserved by OS. This is used to detect rogue processes and to report
+# available memory on a node.
+ducc.agent.node.metrics.sys.gid.max = 500
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: 30000
+ducc.agent.node.metrics.publish.rate = 10000
+
+ducc.agent.node.metrics.endpoint = ducc.node.metrics
+
+# endpoint type choices[topic]
+ducc.agent.node.metrics.endpoint.type = topic
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value: 10000
+ducc.agent.node.inventory.publish.rate = 5000
+
+# If no changes in inventory, publish every 3 intervals defined by
ducc.agent.node.inventory.publish.rate
+ducc.agent.node.inventory.publish.rate.skip = 3
+
+ducc.agent.node.inventory.endpoint = ducc.node.inventory
+
+# endpoint type choices[topic]
+ducc.agent.node.inventory.endpoint.type = topic
+
+# ducc.agent.launcher.thread.pool.size - Deprecated
+ducc.agent.launcher.thread.pool.size = 10
+
+# enable/disable use of ducc_ling
+# ducc.agent.launcher.use.ducc_spawn: true, use ducc_ling. Default: false
+ducc.agent.launcher.use.ducc_spawn = true
+
+# specify location of ducc_ling in the filesystem
+ducc.agent.launcher.ducc_spawn_path = ${DUCC_HOME}/admin/${os.arch}/ducc_ling
+
+# Max amount of time (in millis) agent allows the process to stop before
issuing kill -9
+ducc.agent.launcher.process.stop.timeout = 60000
+
+# Max time in millis allowed for AE initialization. Default 2 hours 7200000.
+ducc.agent.launcher.process.init.timeout = 7200000
+
+# exclude the following user ids while detecting rogue processes
+ducc.agent.rogue.process.user.exclusion.filter =
+
+#exclude the following processes while detecting rogue processes
+ducc.agent.rogue.process.exclusion.filter =
sshd:,-bash,-sh,/bin/sh,/bin/bash,grep,ps
+
+# Fudge Factor (in terms of percantage) that agent uses to multiply a share
size when
+# determining if a JP exceeds its alloted memory, which is calculated as
follows
+# (fudge factor/100)*share size + share size. If this number exceeds JPs RSS,
the agent
+# kills the process.
+ducc.agent.share.size.fudge.factor = 5
+
+# Enable/Disable CGroup support.
+ducc.agent.launcher.cgroups.enable = true
+
+# Define location of cgroups utils like cgexec. For mixed clusters with
different
+# flavors of unix, the utilities may be in different places. In such case, add
a
+# a comma separated list of paths like this:
+# ducc.agent.launcher.cgroups.utils.dir=/usr/bin,<OS 1 path>,<OS 2 path>
+ducc.agent.launcher.cgroups.utils.dir = /usr/bin,/bin
+
+# exclusion file to enable node based exclusion for cgroups and aps
+# syntax: <node>=cgroups,ap
+# the above will exclude node from using cgroups and/or prevent deployment of
APs
+ducc.agent.exclusion.file = ${DUCC_HOME}/resources/exclusion.nodes
+
+# Define cgroup control subsystems used to enforce fair share on a node
+ducc.agent.launcher.cgroups.subsystems = memory,cpu
+
+# Define script which will collect total swap used by a given process. This
+# script is launched by an agent via duccling and running as the owner
+# of the process.
+ducc.agent.swap.usage.script = ${DUCC_HOME}/bin/ducc_get_process_swap_usage.sh
+
+#
+# Uncomment the following line to support auto reaping of rogue processes by
Ducc's Agent
+# ducc.agent.rogue.process.reaper.script=${DUCC_HOME}/admin/ducc_reap_rogues.sh
+#
+# Process Manager Configuration block
+ducc.pm.configuration.class =
org.apache.uima.ducc.pm.config.ProcessManagerConfiguration
+
+ducc.pm.request.endpoint = ducc.pm
+
+# endpoint type choices[queue]
+ducc.pm.request.endpoint.type = queue
+
+#------------------------------------------------------------------------------
+ducc.pm.state.update.endpoint = ducc.pm.state
+
+# endpoint type choices[topic]
+ducc.pm.state.update.endpoint.type = topic
+
+ducc.pm.state.publish.rate = 15000
+
+# New Configuration classes supporting the DUCC 2.0 isolation from user code.
+ducc.uima-as.configuration.class =
org.apache.uima.ducc.transport.configuration.jp.JobProcessConfiguration
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value:
org.apache.uima.ducc.transport.configuration.jp.JobProcessConfiguration
+ducc.job-process.configuration.class =
org.apache.uima.ducc.transport.configuration.jp.JobProcessConfiguration
+
+# OVERRIDE by merger Fri Sep 11 17:17:08 2015 old value:
org.apache.uima.ducc.transport.configuration.service.ServiceConfiguration
+ducc.service.configuration.class =
org.apache.uima.ducc.transport.configuration.service.ServiceConfiguration
+
+ducc.uima-as.endpoint = ducc.job.managed.service
+
+# endpoint type choices[socket]
+ducc.uima-as.endpoint.type = socket
+
+# endpoint parameters which are transport specific. For socket
+# transport params are:
+# - transferExchange=true - include Camel Exchange wrapper in a message
+# - synch=false - use socket transport for one-way messaging (no replies
neeeded)
+ducc.uima-as.endpoint.params = transferExchange=true&sync=false
+
+# saxon8.jar - Saxon is an XSLT and XQuery processor. It is used by UIMA-AS
+# when deploying a service.
+ducc.uima-as.saxon.jar.path = file:${DUCC_HOME}/apache-uima/saxon/saxon8.jar
+
+# dd2spring.xsl - contains a set of rules used when transforming
+# UIMA-AS deployment descriptor into SpringFramework's
+# context file used to deploy UIMA-AS based service.
+ducc.uima-as.dd2spring.xsl.path = ${DUCC_HOME}/apache-uima/bin/dd2spring.xsl
+
+# maximum amount of time to wait for a response from the JD. This value
+# is used by the JP when sending requests to the JD.
+ducc.process.request.timeout = 30000
+
+# Define process container class for DD jobs to instantiate and invoke via
reflection.
+# The container provides classpath isolation for user defined analytics.
+# The container is instantiated with classes from a System classloader.
+ducc.process.uima.as.container.class =
org.apache.uima.ducc.user.jp.UimaASProcessContainer
+
+# Define process container class for non-DD jobs to instantiate and invoke via
reflection.
+# The container provides classpath isolation for user defined analytics.
+# The container is instantiated with classes from a System classloader.
+ducc.process.uima.container.class =
org.apache.uima.ducc.user.jp.UimaProcessContainer
+
+# define the sleep time for JP to use when JD sends empty CAS. In this case the
+# JD's CR has processed its collection. The JP threads need to slow down
sending
+# requests
+ducc.process.thread.sleep.time = 3000
+
+# Custom Flow Controller to use for Ducc Job Processes that don't provide a
process_DD descriptor
+ducc.flow-controller.specifier = org.apache.uima.ducc.FlowController
+
+# INSERT by merger Fri Sep 11 17:17:08 2015
+ducc.runmode = Test
+
+# INSERT by merger Fri Sep 11 17:17:08 2015
+ducc.runmode.pw = quack
+
+# INSERT by merger Fri Sep 11 17:17:08 2015
+ducc.db.jvm.args = -Xmx1G
+
+# INSERT by merger Fri Sep 11 17:17:08 2015
+ducc.rm.state.publish.rate = 10000
+
+# INSERT by merger Fri Sep 11 17:17:08 2015
+ducc.authentication.implementer =
org.apache.uima.ducc.ws.authentication.NeverFailAuthenticator
+
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/ducc.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml?rev=1704040&view=auto
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml
Sat Sep 19 14:19:34 2015
@@ -0,0 +1,250 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";>
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- J[%X{JID}] T[%X{TID}] %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="jd-log" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %c{1} - T[%X{TID}]
%X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="jp-log" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %c{1} - T[%X{TID}]
%X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="cli-console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="rmlog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${DUCC_HOME}/logs/rm.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}-
%X{JID} %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="dblog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${DUCC_HOME}/logs/db.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="orlog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${DUCC_HOME}/logs/or.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- %X{JID} %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="smlog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${DUCC_HOME}/logs/sm.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- %X{JID} %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="pmlog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${DUCC_HOME}/logs/pm.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- %X{JID} %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="wslog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${DUCC_HOME}/logs/ws.${NodeName}.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- %X{JID} %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="jdout"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${ducc.process.log.dir}/jd.out.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- T[%X{TID}] %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="jderr"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file" value="${ducc.process.log.dir}/jd.err.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- T[%X{TID}] %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <appender name="agentlog"
class="org.apache.uima.ducc.common.utils.DeferredOpenRollingAppender">
+ <param name="append" value="true"/>
+ <param name="file"
value="${DUCC_HOME}/logs/${NodeName}.${ducc.deploy.components}.log"/>
+ <param name="maxBackupIndex" value="5" />
+ <param name="maxFileSize" value="10MB" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{DATE} %5p %X{COMPONENT}.%c{1}
- J[%X{JID}] T[%X{TID}] %X{METHOD} %m%n"/>
+ </layout>
+ </appender>
+
+ <category name="org.apache.uima.ducc.rm.scheduler" additivity="true">
+ <priority value="debug"/>
+ <appender-ref ref="rmlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.rm.JobManagerConverter"
additivity="true">
+ <priority value="info"/>
+ <appender-ref ref="rmlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.rm.NodeStability" additivity="true">
+ <priority value="debug"/>
+ <appender-ref ref="rmlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.rm.ResourceManagerComponent"
additivity="true">
+ <priority value="info"/>
+ <appender-ref ref="rmlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.db" additivity="true">
+ <priority value="info"/>
+ <appender-ref ref="dblog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.orchestrator" additivity="true">
+ <priority value="off"/>
+ <appender-ref ref="orlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.agent" additivity="false">
+ <priority value="debug"/>
+ <appender-ref ref="agentlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.agent.deploy" additivity="false">
+ <priority value="off"/>
+ <appender-ref ref="agentlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.jd" additivity="false">
+ <priority value="debug"/>
+ <appender-ref ref="jdout" />
+ </category>
+
+ <category name="org.apache.uima.ducc.sm" additivity="true">
+ <priority value="debug"/>
+ <appender-ref ref="smlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.pm" additivity="true">
+ <priority value="debug"/>
+ <appender-ref ref="pmlog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.ws" additivity="true">
+ <priority value="debug"/>
+ <appender-ref ref="wslog" />
+ </category>
+
+ <category name="org.apache.uima.ducc.user.err" additivity="true">
+ <priority value="debug"/>
+ <appender-ref ref="jderr" />
+ </category>
+
+ <category name="org.apache.activemq">
+ <priority value="warn"/>
+ </category>
+
+ <category name="org.springframework">
+ <priority value="warn"/>
+ </category>
+
+ <category name="org.apache.camel">
+ <priority value="warn"/>
+ </category>
+
+ <category name="org.apache.camel.impl.converter">
+ <priority value="error"/>
+ </category>
+
+ <category name="org.apache.uima.ducc.cli" additivity="false">
+ <priority value="info"/>
+ <appender-ref ref="cli-console" />
+ </category>
+
+ <category name="org.apache.uima.ducc.container" additivity="false">
+ <priority value="debug"/>
+ <appender-ref ref="jd-log" />
+ </category>
+
+ <category name="org.apache.uima.ducc.transport.configuration.jd"
additivity="false">
+ <priority value="debug"/>
+ <appender-ref ref="jd-log" />
+ </category>
+
+ <category name="org.apache.uima.ducc.transport.configuration.jp"
additivity="false">
+ <priority value="info"/>
+ <appender-ref ref="jp-log" />
+ </category>
+
+
+ <root>
+ <priority value = "off" />
+ <appender-ref ref="console" />
+ </root>
+
+</log4j:configuration>
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/resources/log4j.xml
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties?rev=1704040&view=auto
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties
(added)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties
Sat Sep 19 14:19:34 2015
@@ -0,0 +1 @@
+seqno=0
\ No newline at end of file
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/resources/ducc_runtime/state/orcherstrator.properties
------------------------------------------------------------------------------
svn:mime-type = text/plain
