svn commit: r1878160 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.js
Author: degenaro
Date: Wed May 27 12:41:11 2020
New Revision: 1878160
URL: http://svn.apache.org/viewvc?rev=1878160=rev
Log:
UIMA-6238 DUCC Web Server (WS) should not be contacted on timer in manual mode
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.js
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.js
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.js?rev=1878160=1878159=1878160=diff
==
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.js
(original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.js Wed May
27 12:41:11 2020
@@ -3356,6 +3356,8 @@ function ducc_refresh(type) {
ducc_refresh_stopped(type);
type = null
}, 1001);
+ducc_alerts();
+ducc_messages();
ducc_refresh_running(type);
ducc_console_exit(fname);
}
@@ -3447,8 +3449,6 @@ function ducc_timed_loop(type) {
ducc_console_enter(fname);
try {
ducc_cookies();
- ducc_alerts();
- ducc_messages();
var refreshmode = ducc_appl("refreshmode");
var c_value = ducc_get_cookie(refreshmode);
if (c_value == null) {
svn commit: r1874795 - /uima/uima-ducc/trunk/src/main/scripts/ducc_watcher
Author: degenaro
Date: Wed Mar 4 13:58:06 2020
New Revision: 1874795
URL: http://svn.apache.org/viewvc?rev=1874795=rev
Log:
UIMA-6197 ducc_watcher should optionally allow SSL certificate validation bypass
Modified:
uima/uima-ducc/trunk/src/main/scripts/ducc_watcher
Modified: uima/uima-ducc/trunk/src/main/scripts/ducc_watcher
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/scripts/ducc_watcher?rev=1874795=1874794=1874795=diff
==
--- uima/uima-ducc/trunk/src/main/scripts/ducc_watcher (original)
+++ uima/uima-ducc/trunk/src/main/scripts/ducc_watcher Wed Mar 4 13:58:06 2020
@@ -35,6 +35,7 @@ import logging.handlers
import os
import smtplib
import socket
+import ssl
import string
import sys
import time
@@ -79,6 +80,8 @@ state_file = None
flag_agents = False
flag_verbose = False
+flag_ssl_bypass = False
+
mail_host = 'localhost'
email_list = None
@@ -197,6 +200,12 @@ def validate_verbose(options):
if(options.flag_verbose):
flag_verbose = True
+# bypass ssl certification verfication
+def validate_ssl_bypass(options):
+global flag_ssl_bypass
+if(options.flag_ssl_bypass):
+flag_ssl_bypass = True
+
# ignore job driver allocation
# unless --job-driver-allocation is specified
def validate_job_driver_allocation(options):
@@ -304,6 +313,8 @@ def parse_cmdline():
help='mail host (default='+mail_host+')')
parser.add_option('-p','--path', action='store', dest='path', default=None,
help='path to directory where log and state
information are written, default is /tmp'+'/'+get_user())
+parser.add_option('-s','--ssl-bypass', action='store_true',
dest='flag_ssl_bypass', default=False,
+ help='bypass SSL certificate verification (not
recommended)')
parser.add_option('-t','--target', action='store', dest='target',
default=None,
help='[REQUIRED] with default port of
'+port+' or : or http://: or https://:')
parser.add_option('-v','--verbose', action='store_true',
dest='flag_verbose', default=False,
@@ -330,6 +341,8 @@ def parse_cmdline():
validate_agents(options)
# -v
validate_verbose(options)
+# -s
+validate_ssl_bypass(options)
# -j
validate_job_driver_allocation(options)
@@ -457,6 +470,16 @@ def elapsedMillis(dts):
debug(text)
return diff
+def open_url(url):
+global url_timeout
+global flag_ssl_bypass
+if(flag_ssl_bypass and url.startswith('https')):
+ context = ssl._create_unverified_context()
+ response = urllib2.urlopen(url, timeout=url_timeout,
context=context)
+else:
+ response = urllib2.urlopen(url, timeout=url_timeout)
+return response
+
# fetch daemons state
# col[0] = Status
# col[1] = Daemon Name
@@ -469,14 +492,15 @@ def fetch_state_daemons():
global ducc_url_servlet_system_daemons_data
global webserver
global webserver_lifetime_millis
-global url_timeout
+
state_dict_current = {}
daemons = {}
try:
opener = urllib2.build_opener()
+urllib2.install_opener(opener)
if(flag_agents):
opener.addheaders.append(('Cookie', 'DUCCagents=show'))
-response = opener.open(ducc_url_servlet_system_daemons_data,
timeout=url_timeout)
+response = open_url(ducc_url_servlet_system_daemons_data)
data = response.read()
jdata = json.loads(data)['aaData']
for row in jdata:
@@ -529,8 +553,9 @@ def fetch_state_job_driver_allocation():
if(job_driver_allocation != None):
if(rm_up()):
opener = urllib2.build_opener()
+urllib2.install_opener(opener)
debug(ducc_url_servlet_reservations_data)
-response = opener.open(ducc_url_servlet_reservations_data,
timeout=url_timeout)
+response = open_url(ducc_url_servlet_reservations_data)
data = response.read()
debug(data)
json_data = json.loads(data)
svn commit: r1874702 - /uima/uima-ducc/trunk/src/main/scripts/ducc_watcher
Author: degenaro
Date: Mon Mar 2 21:28:55 2020
New Revision: 1874702
URL: http://svn.apache.org/viewvc?rev=1874702=rev
Log:
UIMA-6196 ducc_watcher should support https protocol
Modified:
uima/uima-ducc/trunk/src/main/scripts/ducc_watcher
Modified: uima/uima-ducc/trunk/src/main/scripts/ducc_watcher
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/scripts/ducc_watcher?rev=1874702=1874701=1874702=diff
==
--- uima/uima-ducc/trunk/src/main/scripts/ducc_watcher (original)
+++ uima/uima-ducc/trunk/src/main/scripts/ducc_watcher Mon Mar 2 21:28:55 2020
@@ -245,15 +245,21 @@ def validate_target(options):
global ducc_url_base
global ducc_url_servlet_system_daemons_data
global ducc_url_servlet_reservations_data
-protocol = 'http://'
+protocol_https = 'https://'
+protocol_http = 'http://'
+protocol = protocol_http
if(options.target == None):
error('required "target" not specified')
_exit(1)
target = options.target
if(':' not in target):
target = target+':'+str(port)
-if(target.startswith(protocol)):
-target = target.replace(protocol,'',1)
+if(target.startswith(protocol_https)):
+target = target.replace(protocol_https,'',1)
+protocol = protocol_https
+elif(target.startswith(protocol_http)):
+target = target.replace(protocol_http,'',1)
+#protocol = protocol_http
ducc_url_base = protocol+target
#
servlet = '/ducc-servlet/json-format-aaData-daemons'
@@ -299,7 +305,7 @@ def parse_cmdline():
parser.add_option('-p','--path', action='store', dest='path', default=None,
help='path to directory where log and state
information are written, default is /tmp'+'/'+get_user())
parser.add_option('-t','--target', action='store', dest='target',
default=None,
- help='[REQUIRED] with default port of
'+port+' or :')
+ help='[REQUIRED] with default port of
'+port+' or : or http://: or https://:')
parser.add_option('-v','--verbose', action='store_true',
dest='flag_verbose', default=False,
help='do not reduce noise (in log file)')
svn commit: r1871862 - in /uima/uima-ducc/trunk: src/main/scripts/ uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/ uima-ducc-orchestrato
Author: degenaro Date: Fri Dec 20 14:56:46 2019 New Revision: 1871862 URL: http://svn.apache.org/viewvc?rev=1871862=rev Log: UIMA-6161 DUCC facilitate specification of machine list for user submitted work Modified: uima/uima-ducc/trunk/src/main/scripts/viaducc uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccManagedReservationSubmit.java uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccReservationSubmit.java uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ReservationFactory.java uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/SpecificationProperties.java uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java Modified: uima/uima-ducc/trunk/src/main/scripts/viaducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/scripts/viaducc?rev=1871862=1871861=1871862=diff == --- uima/uima-ducc/trunk/src/main/scripts/viaducc (original) +++ uima/uima-ducc/trunk/src/main/scripts/viaducc Fri Dec 20 14:56:46 2019 @@ -37,6 +37,7 @@ global default_jvm global ducc_home ducc_class = 'fixed' +ducc_mach_list = None java_cmd = None ducc_home = None # tbd in a minute ducc_mem_size = None @@ -70,6 +71,10 @@ def usage(): print " -DDUCC_CLASS=ducc-scheduling-class" print " The default is -DDUCC_CLASS=" + ducc_class print "" +print " -DDUCC_MACHINE_LIST=list-of-machine-names" +print " The default is no machine names. The machine list string should be" +print ' blank-delimeted and quoted, for example: -DDUCC_MACHINE_LIST="machine01 machine02 machine03"' +print "" print " -DDUCC_ENVIRONMENT=environment-settings" print " The default is no additional environment. The environment string should be" print ' blank-delimeted and quoted, for example: -DDUCC_ENVIRONMENT="A=B C=D"' @@ -209,6 +214,7 @@ def main(): usage() ducc_class = 'fixed' +ducc_mach_list = None java_cmd = None ducc_env = '' enable_attach = False @@ -217,6 +223,7 @@ def main(): # remember to add the '=' at the end if following value p_mem_size = '-DDUCC_MEMORY_SIZE=' p_class = '-DDUCC_CLASS=' +p_mach_list = '-DDUCC_MACHINE_LIST=' p_jvm_dir = '-DJAVA_BIN=' p_ducc_home = '-DDUCC_HOME=' p_env = '-DDUCC_ENVIRONMENT=' @@ -245,6 +252,9 @@ def main(): elif (arg.startswith(p_class) ): ducc_class = arg[len(p_class):] +elif (arg.startswith(p_mach_list) ): +ducc_mach_list = '"'+arg[len(p_mach_list):]+'"' + elif (arg.startswith(p_jvm_dir) ): java_cmd = arg[len(p_jvm_dir):] + '/java' @@ -302,6 +312,9 @@ def main(): CMD = CMD + ' --wait_for_completion' CMD = CMD + ' --process_memory_size ' + str(ducc_mem_size) CMD = CMD + ' --scheduling_class ' + ducc_class + +if(ducc_mach_list != None): + CMD = CMD + ' --machine_list ' + ducc_mach_list if ducc_desc_set: CMD = CMD + ' --description %r' % ducc_desc Modified: uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java?rev=1871862=1871861=1871862=diff =
svn commit: r1871703 - /uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/XStreamUtils.java
Author: degenaro
Date: Tue Dec 17 18:14:28 2019
New Revision: 1871703
URL: http://svn.apache.org/viewvc?rev=1871703=rev
Log:
UIMA-6163 DUCC Agent unable to accept Orchestrator publication when new field
added
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/XStreamUtils.java
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/XStreamUtils.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/XStreamUtils.java?rev=1871703=1871702=1871703=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/XStreamUtils.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/XStreamUtils.java
Tue Dec 17 18:14:28 2019
@@ -33,6 +33,7 @@ public class XStreamUtils {
public static String marshall( Object targetToMarshall) throws
Exception {
synchronized(XStreamUtils.class) {
XStream xStream = new XStream(new DomDriver());
+ xStream.ignoreUnknownElements();
initXStreanSecurity(xStream);
return xStream.toXML(targetToMarshall);
}
@@ -40,6 +41,7 @@ public class XStreamUtils {
public static Object unmarshall( String targetToUnmarshall) throws
Exception {
synchronized(XStreamUtils.class) {
XStream xStream = new XStream(new DomDriver());
+ xStream.ignoreUnknownElements();
initXStreanSecurity(xStream);
return xStream.fromXML(targetToUnmarshall);
}
svn commit: r1871334 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Author: degenaro
Date: Fri Dec 13 14:46:51 2019
New Revision: 1871334
URL: http://svn.apache.org/viewvc?rev=1871334=rev
Log:
UIMA-6158 DUCC Web Server (WS) should display Agent BootTime, IP, PID, and
JConsoleUrl when no shared filesystem
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1871334=1871333=1871334=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Fri Dec 13 14:46:51 2019
@@ -391,7 +391,7 @@ public class DuccMachinesData {
return cpu;
}
- private SimpleDateFormat sdf = new SimpleDateFormat(".mm.dd
HH:mm:ss EEE");
+ private SimpleDateFormat sdf = new SimpleDateFormat(".MM.dd
HH:mm:ss EEE");
private long toMillis(String sdate) {
long retVal = 0;
svn commit: r1869433 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMonitor.java
Author: degenaro
Date: Tue Nov 5 21:31:34 2019
New Revision: 1869433
URL: http://svn.apache.org/viewvc?rev=1869433=rev
Log:
UIMA-6073 DUCC Web Server (WS) should discard "down" machines from machines page
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMonitor.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMonitor.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMonitor.java?rev=1869433=1869432=1869433=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMonitor.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMonitor.java
Tue Nov 5 21:31:34 2019
@@ -43,8 +43,8 @@ public class DuccMonitor extends Thread
String location = "turn_on";
try {
if(instance == null) {
- instance.no_kill_i.set(true);
instance = new DuccMonitor();
+ instance.no_kill_i.set(true);
instance.start();
logger.info(location, jobid, "");
}
@@ -61,9 +61,12 @@ public class DuccMonitor extends Thread
public static void turn_off() {
String location = "turn_off";
try {
- instance.no_kill_i.set(false);
- instance = null;
- logger.info(location, jobid, "");
+ if(instance != null) {
+ instance.no_kill_i.set(false);
+ instance = null;
+ logger.info(location, jobid, "");
+ }
+ logger.debug(location, jobid, "");
}
catch(Exception e) {
logger.error(location, jobid, e);
svn commit: r1869292 - in /uima/uima-ducc/trunk: uima-ducc-examples/src/main/scripts/start_sim uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/BasicsHelper.java uima-ducc-web/src/main/java/o
Author: degenaro
Date: Sat Nov 2 11:06:06 2019
New Revision: 1869292
URL: http://svn.apache.org/viewvc?rev=1869292=rev
Log:
UIMA-6073 DUCC Web Server (WS) should discard "down" machines from machines page
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/BasicsHelper.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/DnsHelper.java
Modified: uima/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim?rev=1869292=1869291=1869292=diff
==
--- uima/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim
(original)
+++ uima/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim Sat Nov
2 11:06:06 2019
@@ -316,6 +316,8 @@ class StartSim(DuccUtil):
if ( len(argv) == 0 ):
self.usage(None)
+os.environ['DUCC_SIM'] = 'True'
+
node_config = None
components = {}
instances = {}
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/BasicsHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/BasicsHelper.java?rev=1869292=1869291=1869292=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/BasicsHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/BasicsHelper.java
Sat Nov 2 11:06:06 2019
@@ -41,4 +41,17 @@ public class BasicsHelper {
}
return retVal;
}
+
+ public static boolean isSim() {
+ String location = "isSim";
+ DuccId jobid = null;
+ boolean retVal = false;
+ String ducc_sim_env = System.getenv("DUCC_SIM");
+ if(ducc_sim_env != null) {
+ logger.debug(location, jobid, ducc_sim_env);
+ Boolean bool = new Boolean(ducc_sim_env);
+ retVal = bool.booleanValue();
+ }
+ return retVal;
+ }
}
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/DnsHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/DnsHelper.java?rev=1869292=1869291=1869292=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/DnsHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/DnsHelper.java
Sat Nov 2 11:06:06 2019
@@ -28,7 +28,7 @@ public class DnsHelper {
private static DuccLogger duccLogger =
DuccLogger.getLogger(DnsHelper.class);
- public static boolean isKnowable() {
+ private static boolean isKnowable() {
String location = "isKnowable";
DuccId jobid = null;
boolean retVal = false;
@@ -50,7 +50,7 @@ public class DnsHelper {
// 0 = unknown
// 1 = known
- public static int isKnownHost(String hostname) {
+ private static int isKnownHost(String hostname) {
String location = "isKnownHost";
DuccId jobid = null;
int retVal = -1;
@@ -75,9 +75,14 @@ public class DnsHelper {
public static boolean isKnownHost(NodeId nodeId) {
boolean retVal = true;
- if(nodeId != null) {
- if(isKnownHost(nodeId.getLongName()) == 0) {
- retVal = false;
+ if(BasicsHelper.isSim()) {
+ retVal = true;
+ }
+ else {
+ if(nodeId != null) {
+ if(isKnownHost(nodeId.getLongName()) == 0) {
+ retVal = false;
+ }
}
}
return retVal;
svn commit: r1869274 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Author: degenaro
Date: Fri Nov 1 18:17:28 2019
New Revision: 1869274
URL: http://svn.apache.org/viewvc?rev=1869274=rev
Log:
UIMA-6014 DUCC site.ducc.properties should allow override of DUCC_HOME for WS
display
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1869274=1869273=1869274=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Fri Nov 1 18:17:28 2019
@@ -307,6 +307,9 @@ public class DuccHandler extends DuccAbs
if(text == null) {
text = dir_home;
}
+ else if(text.trim().isEmpty()) {
+ text = dir_home;
+ }
sb.append(text);
response.getWriter().println(sb);
duccLogger.trace(methodName, null, messages.fetch("exit"));
svn commit: r1869261 - in /uima/uima-ducc/trunk: uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/
Author: degenaro
Date: Fri Nov 1 14:03:25 2019
New Revision: 1869261
URL: http://svn.apache.org/viewvc?rev=1869261=rev
Log:
UIMA-6142 DUCC Orchestrator (OR) not killing Job when first JP fails to
initialize
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java?rev=1869261=1869260=1869261=diff
==
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
Fri Nov 1 14:03:25 2019
@@ -18,6 +18,7 @@
*/
package org.apache.uima.ducc.orchestrator;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@@ -1538,6 +1539,26 @@ public class StateManager {
logger.trace(methodName, null, messages.fetch("exit"));
}
+ private DuccId getFirstFailedInitialization(DuccWorkJob job) {
+ String methodName = "getFirstFailedInitialization";
+ DuccId retVal = null;
+ try {
+ IDuccProcessMap processMap = job.getProcessMap();
+ ArrayList list =
processMap.getFailedInitialization();
+ for(DuccId duccId : list) {
+ IDuccProcess process =
processMap.getProcess(duccId);
+ if(process.getSeqNo() == 0) {
+ retVal = duccId;
+ break;
+ }
+ }
+ }
+ catch(Throwable t) {
+ logger.error(methodName, null, t);
+ }
+ return retVal;
+ }
+
private void inventoryJob(IDuccWork duccWork, IDuccProcess
inventoryProcess) {
String methodName = "inventoryJob";
DuccWorkJob job = (DuccWorkJob) duccWork;
@@ -1600,7 +1621,16 @@ public class StateManager {
}
long initFailureCount =
job.getProcessInitFailureCount();
long startup_initialization_error_limit
=
DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_jd_startup_initialization_error_limit,
1);
- if(initFailureCount >=
startup_initialization_error_limit) {
+ DuccId pid =
getFirstFailedInitialization(job);
+ if(pid != null) {
+ String reason = "process
inititialization failure on first job process";
+ logger.warn(methodName,
job.getDuccId(), pid, reason);
+ JobCompletionType
jobCompletionType = JobCompletionType.CanceledBySystem;
+ Rationale rationale = new
Rationale(reason);
+ ProcessDeallocationType
processDeallocationType = ProcessDeallocationType.JobCanceled;
+ stateManager.jobTerminate(job,
jobCompletionType, rationale, processDeallocationType);
+ }
+ else if(initFailureCount >=
startup_initialization_error_limit) {
String reason = "process
inititialization failure count["+initFailureCount+"] meets startup
initialization error limit["+startup_initialization_error_limit+"]";
logger.warn(methodName,
job.getDuccId(), reason);
JobCompletionType
jobCompletionType = JobCompletionType.CanceledBySystem;
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java?rev=1869261=1869260=1869261=diff
svn commit: r1868195 - /uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Author: degenaro
Date: Wed Oct 9 14:44:37 2019
New Revision: 1868195
URL: http://svn.apache.org/viewvc?rev=1868195=rev
Log:
UIMA-6034 DUCC Orchestrator (OR) request validation
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java?rev=1868195=1868194=1868195=diff
==
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Wed Oct 9 14:44:37 2019
@@ -849,7 +849,7 @@ implements Orchestrator {
}
private boolean isSignatureValid(Properties properties) {
String methodName = "isSignatureValid";
- boolean retVal = true;
+ boolean retVal = false;
try {
if(orchestratorCommonArea.isSignatureRequired()) {
// Check that the signature is valid
svn commit: r1868154 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws: helper/DiagnosticsHelper.java server/AlienTextFile.java server/DuccHandler.java
Author: degenaro
Date: Tue Oct 8 19:25:32 2019
New Revision: 1868154
URL: http://svn.apache.org/viewvc?rev=1868154=rev
Log:
UIMA-6014 DUCC site.ducc.properties should allow override of DUCC_HOME for WS
display
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/DiagnosticsHelper.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/AlienTextFile.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/DiagnosticsHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/DiagnosticsHelper.java?rev=1868154=1868153=1868154=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/DiagnosticsHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/DiagnosticsHelper.java
Tue Oct 8 19:25:32 2019
@@ -29,7 +29,7 @@ import org.apache.uima.ducc.common.utils
public class DiagnosticsHelper extends Thread {
- private static DuccLogger duccLogger =
DuccLogger.getLogger(DiagnosticsHelper .class.getName(), null);
+ private static DuccLogger duccLogger =
DuccLogger.getLogger(DiagnosticsHelper .class.getName(), "WS");
private static DuccId jobid = null;
private static File devNull = new File("/dev/null");
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/AlienTextFile.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/AlienTextFile.java?rev=1868154=1868153=1868154=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/AlienTextFile.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/AlienTextFile.java
Tue Oct 8 19:25:32 2019
@@ -31,7 +31,7 @@ import org.apache.uima.ducc.ws.utils.ali
public class AlienTextFile extends AlienAbstract {
- private static DuccLogger duccLogger =
DuccLogger.getLogger(AlienTextFile.class.getName(), null);
+ private static DuccLogger duccLogger =
DuccLogger.getLogger(AlienTextFile.class.getName(), "WS");
private static String command_du = "/usr/bin/du";
private static String flag_dash_b = "-b";
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1868154=1868153=1868154=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Tue Oct 8 19:25:32 2019
@@ -302,7 +302,12 @@ public class DuccHandler extends DuccAbs
String methodName = "handleDuccServletHome";
duccLogger.trace(methodName, null, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
- sb.append(dir_home);
+ String ducc_ws_display_home = "ducc.ws.display.home";
+ String text =
DuccPropertiesResolver.getInstance().getProperty(ducc_ws_display_home);
+ if(text == null) {
+ text = dir_home;
+ }
+ sb.append(text);
response.getWriter().println(sb);
duccLogger.trace(methodName, null, messages.fetch("exit"));
}
svn propchange: r1867147 - svn:log
Author: degenaro Revision: 1867147 Modified property: svn:log Modified: svn:log at Wed Sep 18 20:38:58 2019 -- --- svn:log (original) +++ svn:log Wed Sep 18 20:38:58 2019 @@ -0,0 +1 @@ +fix complaint about db_pidfile
svn commit: r1867147 - /uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
Author: degenaro Date: Wed Sep 18 20:34:44 2019 New Revision: 1867147 URL: http://svn.apache.org/viewvc?rev=1867147=rev Log: (empty) Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1867147=1867146=1867147=diff == --- uima/uima-ducc/trunk/src/main/resources/default.ducc.properties (original) +++ uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Wed Sep 18 20:34:44 2019 @@ -296,6 +296,9 @@ ducc.ws.port.ssl = 42155 # Optionally configure the Web Server welcome page, default is index.html (which forwards to jobs.html) ducc.ws.welcome.page = index.html +# Optionally configure the Web Server DUCC_HOME display value, default is absolute path of DUCC_HOME. +ducc.ws.display.home = + # Optionally configure the Web Server job automatic cancel timeout, default is 10. To disable # feature specify 0. Employed when user specifies --wait_for_completion flag on job submission, # in which case the job monitor program must visit
svn commit: r1867146 - /uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Author: degenaro Date: Wed Sep 18 20:34:12 2019 New Revision: 1867146 URL: http://svn.apache.org/viewvc?rev=1867146=rev Log: UIMA-6125 ducc_update fails when DB automange is false but DB is running Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1867146=1867145=1867146=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Sep 18 20:34:12 2019 @@ -292,6 +292,8 @@ class DuccUtil(DuccBase): # does the database process exist? def db_process_alive(self): +if (not self.automanage_database): +return False if ( not os.path.exists(self.db_pidfile) ): return False @@ -312,6 +314,8 @@ class DuccUtil(DuccBase): return False def db_process_kill(self,code): +if (not self.automanage_database): +return False if ( not os.path.exists(self.db_pidfile) ): return False @@ -349,7 +353,9 @@ class DuccUtil(DuccBase): if(verbose): print 'No database location defined.' return False - +if (not self.automanage_database): +return False + if ( not os.path.exists(self.db_pidfile) ): if(verbose): print 'Database pid file does not exist. Checking DB connectivity.'
svn commit: r1867113 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/etc/http-uri-encryption-exemption.list
Author: degenaro Date: Wed Sep 18 12:57:41 2019 New Revision: 1867113 URL: http://svn.apache.org/viewvc?rev=1867113=rev Log: UIMA-6127 DUCC Web Server (WS) should allow duccinator-update request via http Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/etc/http-uri-encryption-exemption.list Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/etc/http-uri-encryption-exemption.list URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/etc/http-uri-encryption-exemption.list?rev=1867113=1867112=1867113=diff == --- uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/etc/http-uri-encryption-exemption.list (original) +++ uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/etc/http-uri-encryption-exemption.list Wed Sep 18 12:57:41 2019 @@ -14,6 +14,7 @@ /ducc-servlet/login-link /ducc-servlet/logout-link /ducc-servlet/authenticator-password-checked +/ducc-servlet/duccinator-update /favicon.ico /uima.ico /login.html
svn commit: r1866794 - in /uima/uima-ducc/trunk/src/main/admin: check_ducc ducc.py start_ducc
Author: degenaro
Date: Wed Sep 11 13:01:23 2019
New Revision: 1866794
URL: http://svn.apache.org/viewvc?rev=1866794=rev
Log:
UIMA-6125 ducc_update fails when DB automange is false but DB is running
Modified:
uima/uima-ducc/trunk/src/main/admin/check_ducc
uima/uima-ducc/trunk/src/main/admin/ducc.py
uima/uima-ducc/trunk/src/main/admin/start_ducc
Modified: uima/uima-ducc/trunk/src/main/admin/check_ducc
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/check_ducc?rev=1866794=1866793=1866794=diff
==
--- uima/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/uima-ducc/trunk/src/main/admin/check_ducc Wed Sep 11 13:01:23 2019
@@ -40,7 +40,10 @@ class CheckDucc(DuccUtil):
self.badnodes = []
def validate(self, checkdate):
-verify_slave_node(checkdate, self.ducc_properties)
+ if(self.is_head_node()):
+ pass
+ else:
+ verify_slave_node(checkdate, self.ducc_properties)
self.check_clock_skew(checkdate)
self.verify_jvm()
self.verify_limits()
@@ -312,9 +315,10 @@ class CheckDucc(DuccUtil):
if ( jvm == None ):
print 'WARN: ducc.jvm is not specified in ducc.properties. Default
is simply "java" which may not work on all nodes.'
-if ( not verify_master_node(self.ducc_properties) ):
-print 'FAIL: Cannot verify master mode'
-return
+ if(self.is_head_node()):
+ if ( not verify_master_node(self.ducc_properties) ):
+ print 'FAIL: Cannot verify master mode'
+ return
if ( not self.verify_activemq() ):
print 'ActiveMQ broker is not running on', self.broker_protocol +
"://" + self.broker_host + ':' + self.broker_port
Modified: uima/uima-ducc/trunk/src/main/admin/ducc.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc.py?rev=1866794=1866793=1866794=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc.py Wed Sep 11 13:01:23 2019
@@ -174,13 +174,15 @@ class Ducc(DuccUtil):
print 'NOT_OK Cannot proceed because of ducc_ling
problems.'
return
-if ( not verify_slave_node(localdate, self.ducc_properties) ):
-# we assume that verify_local_node is spewing a line of
the form
-#NOTOK error message
-# if all is not fine
-print '0 ONE RETURNS'
-
-return
+if(self.is_head_node()):
+pass
+else:
+if ( not verify_slave_node(localdate,
self.ducc_properties) ):
+# we assume that verify_local_node is spewing a line
of the form
+#NOTOK error message
+# if all is not fine
+print '0 ONE RETURNS'
+return
jvm_opts.append('-Djava.library.path=' + ducc_home)
if ( self.agent_jvm_args != None ):
Modified: uima/uima-ducc/trunk/src/main/admin/start_ducc
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/start_ducc?rev=1866794=1866793=1866794=diff
==
--- uima/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/uima-ducc/trunk/src/main/admin/start_ducc Wed Sep 11 13:01:23 2019
@@ -306,9 +306,10 @@ class StartDucc(DuccUtil):
self.verify_required_directories()
# If this fails local_hooks.py should report the reason
-if ( not verify_master_node(self.ducc_properties) ):
-return
-
+if(self.is_head_node()):
+if ( not verify_master_node(self.ducc_properties) ):
+return
+
# make sure all the nodefiles exist and are readable
ok = True
nodes = {}
svn commit: r1866413 - /uima/uima-ducc/trunk/src/main/admin/start_ducc
Author: degenaro
Date: Wed Sep 4 17:59:19 2019
New Revision: 1866413
URL: http://svn.apache.org/viewvc?rev=1866413=rev
Log:
UIMA-6123 DUCC start_ducc should display NOTOK messages
Modified:
uima/uima-ducc/trunk/src/main/admin/start_ducc
Modified: uima/uima-ducc/trunk/src/main/admin/start_ducc
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/start_ducc?rev=1866413=1866412=1866413=diff
==
--- uima/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/uima-ducc/trunk/src/main/admin/start_ducc Wed Sep 4 17:59:19 2019
@@ -127,7 +127,10 @@ class StartDucc(DuccUtil):
if ( line.startswith('WARN') ):
msgs.append(('', line))
pid = '?'
-
+if ( line.startswith('NOTOK') ):
+msgs.append(('', line))
+pid = '?'
+
sshmsgs = self.ssh_ok(node, line )
if ( sshmsgs != None ):
for m in sshmsgs:
svn commit: r1866390 - /uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
Author: degenaro
Date: Wed Sep 4 13:51:33 2019
New Revision: 1866390
URL: http://svn.apache.org/viewvc?rev=1866390=rev
Log:
UIMA-6121 DUCC Book - review and update shared filesystem requirements,
especially for Agents
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex?rev=1866390=1866389=1866390=diff
==
---
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
(original)
+++
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/install.tex
Wed Sep 4 13:51:33 2019
@@ -93,10 +93,10 @@ Single system installation:
Additional requirements for multiple system installation:
-\begin{itemize}
- \item All systems must have a shared filesystem (such as NFS or GPFS) and
common user credentials.
-The \$DUCC\_HOME directory must be located on a shared filesystem.
-\end{itemize}
+There are two separate modes of operation: using a shared filesystem or not.
+In both cases all host systems must have common user credentials (e.g.
userids, groupids).
+If using a shared filesystem then it too must have common user credentials and
the
+the \$DUCC\_HOME directory must be located there.
Additional requirements for running multiple user processes with their own
credentials.
@@ -207,13 +207,14 @@ If building from source, Maven places th
Any user ID can be used to run a single-system DUCC, but ``ducc'' userid is
recommended.
This user will employ cgroups as well as launching and running processes with
the credentials of the submitting user.
-If multiple nodes are going to be added later, the ducc runtime tree should be
installed
+If multiple nodes are going to be added later, the ducc runtime tree can be
installed
on a shared filesystem so that it can be mounted on the additional nodes.
+If not using a shared filesystem, ducc\_rsync on the ducc head node can be used
+to propagate the ducc runtime tree to the additional nodes.
Verification submits a very simple UIMA pipeline for execution under DUCC.
Once this is shown to be
working, one may proceed installing additional features.
-
\section{Minimal Hardware Requirements for Single System Installation}
\begin{itemize}
\item One Intel-based or IBM Power-based system (Big or Little Endian).
(More systems may be added later.)
@@ -241,8 +242,8 @@ workloads.
This creates a directory with a name of the form
``apache-uima-ducc-[version]''.
This directory contains the full DUCC runtime which
-you may use ``in place'' but it is highly recommended that you move it
-into a standard location on a shared filesystem; for example, under
ducc's HOME directory:
+you may use ``in place'', or move to a standard location on a shared
filesystem;
+for example, under ducc's HOME directory:
\begin{verbatim}
mv apache-uima-ducc-[version] /home/ducc/ducc_runtime
\end{verbatim}
@@ -400,9 +401,38 @@ $HOME/ducc/logs/job-id
\hyperref[sec:install.prerequisites]{\em prerequisites}.
\fi
- \$DUCC\_HOME must be on a shared filesystem and mounted at the same location
- on all DUCC nodes.
+ \$DUCC\_HOME must be at the same location on all DUCC nodes, and can be on
a shared filesystem.
+ On the additional nodes, setup user ducc .ssh/authorized\_keys to allow
passwordless ssh from the
+ DUCC head node.
+
+ If not using a shared filesystem for \$DUCC\_HOME then use
admin/ducc\_rsync to populate the additional node(s).
+ Once populated, use admin/start\_ducc to start the agent(s) on the
additional node(s).
+
+ Example:
+
+ \begin{verbatim}
+[ducc@ducc-master-3 admin]$ ./ducc_rsync --agent-nodes ducc-worker-901
+ Merging
/opt/ducc/install/ducc/apache-uima-ducc-3.0.0/resources/default.ducc.properties
with
/opt/ducc/install/ducc/apache-uima-ducc-3.0.0/resources/site.ducc.properties
into /opt/ducc/install/ducc/apache-uima-ducc-3.0.0/resources/ducc.properties
+ rsync ducc ducc-worker-901 admin
+ rsync ducc ducc-worker-901 apache-uima
+ rsync ducc ducc-worker-901 bin
+ rsync ducc ducc-worker-901 cron
+ rsync ducc ducc-worker-901 docs
+ rsync ducc ducc-worker-901 duccling
+ rsync ducc ducc-worker-901 examples
+ rsync ducc ducc-worker-901 INSTALL
+ rsync ducc ducc-worker-901 issuesFixed
+ rsync ducc ducc-worker-901 lib
+ rsync ducc ducc-worker-901 LICENSE
+ rsync ducc ducc-worker-901 NOTICE
+ rsync ducc ducc-worker-901 README
+ rsync ducc ducc-worker-901 RELEASE_NOTES.html
+ rsync ducc ducc-worker-901 resources
+ rsync ducc ducc-worker-901
resources.private/ducc-broker-credentials.properties
svn commit: r1866345 - /uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Author: degenaro
Date: Tue Sep 3 20:27:37 2019
New Revision: 1866345
URL: http://svn.apache.org/viewvc?rev=1866345=rev
Log:
UIMA-6122 admin/ducc_util.py should not require ducc.private.properties as
agent has no need
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1866345=1866344=1866345=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Tue Sep 3 20:27:37 2019
@@ -258,17 +258,26 @@ class DuccUtil(DuccBase):
else:
self.db_bypass = False
-dbprops = Properties()
-dbprops.load(self.DUCC_HOME +
'/resources.private/ducc.private.properties')
-self.db_password = dbprops.get('db_password')
-if ( self.db_password == None ):
-print "bypassing database because no password is set."
-self.db_bypass = True
-
-self.db_password_guest = dbprops.get('db_password_guest')
-if ( self.db_password_guest == None ):
-self.db_password_guest = 'guest'
+fdir = 'resources.private'
+fname = 'ducc.private.properties'
+fpath = os.path.join(self.DUCC_HOME,fdir,fname)
+if(os.path.isfile(fpath)):
+dbprops = Properties()
+dbprops.load(fpath)
+self.db_password = dbprops.get('db_password')
+if ( self.db_password == None ):
+print 'bypassing database because no password is set.'
+self.db_bypass = True
+self.db_password_guest = dbprops.get('db_password_guest')
+if ( self.db_password_guest == None ):
+self.db_password_guest = 'guest'
+else:
+self.db_password == None
+self.db_password_guest == None
+print 'bypassing database because '+fname+' not found.'
+self.db_bypass = True
+
def db_password(self):
if(self.db_password == None):
self.db_configure()
svn commit: r1865989 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server: DuccAbstractHandler.java DuccHandlerClassic.java
Author: degenaro
Date: Tue Aug 27 12:40:44 2019
New Revision: 1865989
URL: http://svn.apache.org/viewvc?rev=1865989=rev
Log:
UIMA-6071 DUCC cli ducc_status info is deficient
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java?rev=1865989=1865988=1865989=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
Tue Aug 27 12:40:44 2019
@@ -188,7 +188,7 @@ public abstract class DuccAbstractHandle
public String toDash(String value) {
String retVal = "-";
if(value != null) {
- if(value.length() > 0) {
+ if(value.trim().length() > 0) {
retVal = value;
}
}
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1865989=1865988=1865989=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
Tue Aug 27 12:40:44 2019
@@ -1713,12 +1713,12 @@ public class DuccHandlerClassic extends
// Boot Time
String bootTime =
getTimeStamp(DuccCookies.getDateStyle(request),getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyBootTime,""));
sb.append("");
- sb.append(bootTime);
+ sb.append(toDash(bootTime));
sb.append("");
// Host IP
String hostIP =
getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyNodeIpAddress,"");
sb.append("");
- sb.append(hostIP);
+ sb.append(toDash(hostIP));
sb.append("");
// Host Name
String hostName = machineInfo.getName();
@@ -1728,7 +1728,7 @@ public class DuccHandlerClassic extends
// PID
String pid =
getPropertiesValue(properties,DuccDaemonRuntimeProperties.keyPid,"");
sb.append("");
- sb.append(pid);
+ sb.append(toDash(pid));
sb.append("");
// Publication Size (last)
String publicationSizeLast =
machineInfo.getPublicationSizeLast();
svn commit: r1865988 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
Author: degenaro
Date: Tue Aug 27 12:29:39 2019
New Revision: 1865988
URL: http://svn.apache.org/viewvc?rev=1865988=rev
Log:
UIMA-6117 DUCC Web Server (WS) should display "no data - not master" on Viz page
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java?rev=1865988=1865987=1865988=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
Tue Aug 27 12:29:39 2019
@@ -24,9 +24,11 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.uima.ducc.common.head.IDuccHead;
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.ws.DuccHead;
import org.apache.uima.ducc.ws.server.nodeviz.NodeViz;
import org.eclipse.jetty.server.Request;
@@ -36,6 +38,8 @@ public class DuccHandlerViz extends Ducc
private static Messages messages = Messages.getInstance();
private static DuccId jobid = null;
+ private static IDuccHead dh = DuccHead.getInstance();
+
public final String vizNodes=
duccContextViz+"-nodes";
NodeViz viz = null;
@@ -51,10 +55,21 @@ public class DuccHandlerViz extends Ducc
String methodName = "handleServletVizNodes";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
+ String data = null;
- //String data = ""+methodName+" not yet
implemented";
- String data = viz.getVisualization();
-
+ if(dh.is_ducc_head_backup()) {
+ StringBuffer sb = new StringBuffer();
+ sb.append("");
+ sb.append("");
+ sb.append(messages.fetch("no data - not master"));
+ sb.append("");
+ sb.append("");
+ data = sb.toString();
+ }
+ else {
+ data = viz.getVisualization();
+ }
+
duccLogger.debug(methodName, jobid, data);
response.getWriter().println(data);
duccLogger.trace(methodName, jobid, messages.fetch("exit"));
svn commit: r1865751 - in /uima/uima-ducc/trunk: src/main/admin/ uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/
Author: degenaro Date: Fri Aug 23 14:03:26 2019 New Revision: 1865751 URL: http://svn.apache.org/viewvc?rev=1865751=rev Log: UIMA-6037 agent autostart should rsync its local ducc_runtime tree Added: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerRsync.java (with props) Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync uima/uima-ducc/trunk/src/main/admin/ducc_util.py uima/uima-ducc/trunk/src/main/admin/rm_qoccupancy uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebServer.java Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1865751=1865750=1865751=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Fri Aug 23 14:03:26 2019 @@ -47,7 +47,16 @@ class DuccRsync(DuccUtil): key_ducc_head_reliable_list = 'ducc.head.reliable.list' rsync_cmd = 'rsync' -rsync_flags = '-avz --delete --ignore-errors' +rsync_flags = '-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -avz --links --delete --ignore-errors --timeout 20' + +symlink_cmd = 'rsync' +symlink_flags = '-e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -vz --links --delete --ignore-errors --timeout 20' + +ssh_cmd = 'ssh' +ssh_flags = '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + +mkdir_cmd = 'mkdir' +mkdir_flags = '-p' head_dirs_list = [ 'admin', @@ -93,18 +102,24 @@ class DuccRsync(DuccUtil): 'README', 'RELEASE_NOTES.html', 'resources', -'resources.private/ducc-broker-credentials.properties', +#'resources.private', #'state', -'state/duccling.version', #'webserver', ] +agent_files_list = [ +'resources.private/ducc-broker-credentials.properties', +'state/duccling.version', +] + dual_dirs_list = head_dirs_list head_dirs = ' '.join(head_dirs_list) agent_dirs = ' '.join(agent_dirs_list) dual_dirs = ' '.join(dual_dirs_list) +agent_files = ' '.join(agent_files_list) + def __init__(self): DuccUtil.__init__(self, self.merge) @@ -200,16 +215,20 @@ class DuccRsync(DuccUtil): # update nodes that are head+agent def update_dual(self,user): +rc = 0 if(self.head_nodes != None): if(self.agent_nodes != None): for node in self.head_nodes: if(node in self.agent_nodes): if(self.args.debug): print 'update head+agent: '+node -self.rsync(node,user,self.dual_dirs) +self.mkdir(node,user) +rc = self.rsync_dirs(node,user,self.dual_dirs) +return rc # update nodes that are head only def update_heads(self,user): +rc = 0 update = True if(self.head_nodes != None): for node in self.head_nodes: @@ -220,10 +239,13 @@ class DuccRsync(DuccUtil): if(update): if(self.args.debug): print 'update head: '+node -self.rsync(node,user,self.head_dirs) +self.mkdir(node,user) +rc = self.rsync_dirs(node,user,self.head_dirs) +return rc # update nodes that are agent only def update_agents(self,user): +rc = 0 update = True if(self.agent_nodes != None): for node in self.agent_nodes: @@ -234,10 +256,32 @@ class DuccRsync(DuccUtil): if(update): if(self.args.debug): print 'update agent: '+node -self.rsync(node,user,self.agent_dirs) - -# update the specified node, subdirs -def rsync(self,node,user,subdirs): +self.mkdir(node,user) +rc = self.rsync_dirs(node,user,self.agent_dirs) +if(rc == 0): +rc = self.rsync_files(node,user,self.agent_files) +return rc + +# update the specified node dirs +def mkdir(self,node,user): +rc = 0 +rmt = self.mkdir_cmd+' '+self.mkdir_flags+' '+DUCC_HOME +cmd = self.ssh_cmd+' '+self.ssh_flags+' '+user+'@'+node+' '+rmt +proc = subprocess.Popen(cmd, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +lines = [] +for line in proc.stdout: +lines.append(line.strip()) +proc.wait() +rc = proc.returncode
svn commit: r1864972 - /uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties
Author: degenaro Date: Mon Aug 12 14:42:27 2019 New Revision: 1864972 URL: http://svn.apache.org/viewvc?rev=1864972=rev Log: UIMA-6113 DUCC Apache demo services are killed, causing jobs with dependencies to hang at start-up Modified: uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties Modified: uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties?rev=1864972=1864971=1864972=diff == --- uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties (original) +++ uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties Mon Aug 12 14:42:27 2019 @@ -75,5 +75,5 @@ ducc.rm.fragmentation.threshold = 1 ducc.rm.share.quantum = 15 ducc.agent.rogue.process.user.exclusion.filter=root,postfix,ntp,nobody,daemon,100,degenaro,messagebus,nslcd,syslog,whoopsie,ducc -ducc.agent.node.metrics.publish.rate=1 -ducc.agent.node.inventory.publish.rate=5000 +ducc.agent.node.metrics.publish.rate=3 +ducc.agent.node.inventory.publish.rate=15000
svn commit: r1864509 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server: DuccCookies.java DuccHandler.java DuccHandlerUserAuthentication.java DuccWebSessionManager.j
Author: degenaro
Date: Tue Aug 6 11:22:11 2019
New Revision: 1864509
URL: http://svn.apache.org/viewvc?rev=1864509=rev
Log:
UIMA-6085 DUCC Web Server (WS) login session should be coordinated amongst DUCC
head nodes
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccCookies.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerUserAuthentication.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebSessionManager.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccCookies.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccCookies.java?rev=1864509=1864508=1864509=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccCookies.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccCookies.java
Tue Aug 6 11:22:11 2019
@@ -71,7 +71,8 @@ public class DuccCookies {
private static final String filter_users_style = "filter_users_style";
private static final String role = "role";
- private static final String uid = "uid";
+ private static final String key_uid = "uid";
+ private static final String key_loginToken = "loginToken";
public static final String cookieStyleTable =
duccCookiePrefix+table_style;
public static final String cookieStyleDate =
duccCookiePrefix+date_style;
@@ -80,7 +81,8 @@ public class DuccCookies {
public static final String cookieStyleFilterUsers =
duccCookiePrefix+filter_users_style;
public static final String cookieRole = duccCookiePrefix+role;
- public static final String cookieUid = duccCookiePrefix+uid;
+ public static final String cookieUid = duccCookiePrefix+key_uid;
+ public static final String cookieLoginToken =
duccCookiePrefix+key_loginToken;
public static final String valueStyleDateLong = "long";
public static final String valueStyleDateMedium = "medium";
@@ -103,6 +105,12 @@ public class DuccCookies {
public static final String valueRoleAdministrator = "administrator";
public static final String valueRoleUser = "user";
+ public static final int seconds_per_minute = 60;
+ public static final int seconds_per_hour = 60*seconds_per_minute;
+ public static final int seconds_per_day = 24*seconds_per_hour;
+ public static final int seconds_per_year = 365*seconds_per_day;
+ public static final int seconds_per_century = 100*seconds_per_year;
+
protected static final String getCookieKey(String name) {
return duccCookiePrefix+"name";
}
@@ -137,18 +145,19 @@ public class DuccCookies {
return getCookie(null,request,name);
}
- protected static void putCookie(HttpServletResponse response, String
name, String value) {
+ protected static void putCookie(HttpServletResponse response, String
name, String value, int expiry) {
String methodName = "putCookie";
Cookie cookie = new Cookie(name, value);
cookie.setPath(cookieUri);
+ cookie.setMaxAge(expiry);
response.addCookie(cookie);
duccLogger.trace(methodName, null,
messages.fetchLabel("name")+name+" "+messages.fetchLabel("value")+value);
}
- protected static void expireCookie(HttpServletResponse response, String
name, String value) {
- String methodName = "expireCookie";
+ protected static void putCookie(HttpServletResponse response, String
name, String value) {
+ String methodName = "putCookie";
Cookie cookie = new Cookie(name, value);
- cookie.setMaxAge(0);
+ cookie.setPath(cookieUri);
response.addCookie(cookie);
duccLogger.trace(methodName, null,
messages.fetchLabel("name")+name+" "+messages.fetchLabel("value")+value);
}
@@ -272,16 +281,58 @@ public class DuccCookies {
return role;
}
- public static String getUid(HttpServletRequest request) {
- String location = "getUid";
- String uid = null;
+ public static String getLoginUid(HttpServletRequest request) {
+ String location = "getLoginUid";
+ String loginUid
svn propchange: r1864453 - svn:log
Author: degenaro Revision: 1864453 Modified property: svn:log Modified: svn:log at Tue Aug 6 11:12:08 2019 -- --- svn:log (original) +++ svn:log Tue Aug 6 11:12:08 2019 @@ -0,0 +1 @@ +UIMA-6085 DUCC Web Server (WS) login session should be coordinated amongst DUCC head nodes
svn commit: r1864453 - in /uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login: ./ DbUserLogin.java IDbUserLogin.java
Author: degenaro
Date: Mon Aug 5 18:54:04 2019
New Revision: 1864453
URL: http://svn.apache.org/viewvc?rev=1864453=rev
Log: (empty)
Added:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/DbUserLogin.java
(with props)
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/IDbUserLogin.java
(with props)
Added:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/DbUserLogin.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/DbUserLogin.java?rev=1864453=auto
==
---
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/DbUserLogin.java
(added)
+++
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/login/DbUserLogin.java
Mon Aug 5 18:54:04 2019
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+
+/**
+ * Class to maintain and query database table of login userids.
+ */
+
+package org.apache.uima.ducc.database.login;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.uima.ducc.common.db.DbHelper;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.database.DbHandle;
+import org.apache.uima.ducc.database.DbManager;
+import org.apache.uima.ducc.database.DbUtil;
+
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.InvalidQueryException;
+
+/*
+ * Class to manage table of user logins
+ */
+public class DbUserLogin implements IDbUserLogin {
+
+ private static String KEYSPACE = "DUCC";
+ private static String USER_LOGIN_TABLE =
UserLoginProperties.TABLE_NAME.pname();
+
+ private static String key_name = UserLoginProperties.name.name();
+ private static String key_validationId =
UserLoginProperties.validationId.name();
+
+ private DuccLogger logger = null;
+ private DuccId id = null;
+
+ private DbManager dbManager = null;
+
+ public DbUserLogin(DuccLogger logger) {
+ this.logger = logger;
+ init();
+ }
+
+ private boolean init() {
+ String mn = "init";
+ boolean retVal = false;
+ try {
+ String[] dbUrls = DbHelper.getHostList();
+ StringBuffer sb = new StringBuffer();
+ for(String dbUrl : dbUrls) {
+ sb.append(dbUrl+" ");
+ }
+ logger.debug(mn, id, dbUrls.length+" "+sb.toString());
+ retVal = init(dbUrls);
+ }
+ catch(Exception e) {
+ logger.error(mn, id, e);
+ }
+ db_init();
+ return retVal;
+ }
+
+ private boolean init(String[] dburls) throws Exception {
+ String mn = "init";
+ boolean retVal = false;
+ try {
+ dbManager = new DbManager(dburls, logger);
+ dbManager.init();
+ retVal = true;
+ } catch (Exception e) {
+ logger.error(mn, id, "Errors contacting database. No
connetion made.");
+ logger.error(mn, id, e);
+ }
+ return retVal;
+ }
+
+ private boolean db_init() {
+ String mn = "db_init";
+ boolean retVal = false;
+ try {
+ ListspecificationsSchema =
db_mkSchema();
+ DbH
svn commit: r1863954 - /uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
Author: degenaro
Date: Mon Jul 29 16:13:55 2019
New Revision: 1863954
URL: http://svn.apache.org/viewvc?rev=1863954=rev
Log:
UIMA-6101 DUCC demo on Apache moved to vm3: userid duccy --> ducc
- use hostname+domain for service names
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py?rev=1863954=1863953=1863954=diff
==
---
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
(original)
+++
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
Mon Jul 29 16:13:55 2019
@@ -82,7 +82,8 @@ class Helper():
#
-hostname = socket.gethostname().split('.')[0]
+#hostname = socket.gethostname().split('.')[0]
+hostname = socket.gethostname()
port = '61617'
dictService = {
svn commit: r1863950 - /uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
Author: degenaro
Date: Mon Jul 29 15:23:21 2019
New Revision: 1863950
URL: http://svn.apache.org/viewvc?rev=1863950=rev
Log:
UIMA-6101 DUCC demo on Apache moved to vm3: userid duccy --> ducc
Modified:
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1863950=1863949=1863950=diff
==
---
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
Mon Jul 29 15:23:21 2019
@@ -351,7 +351,7 @@ public class ServiceHandler
DuccId id = w.getDuccId();
String[] deps = w.getServiceDependencies();
-logger.info(methodName, id, deps.length);
+logger.debug(methodName, id, deps.length);
// New services, if any are discovered
// Put them into the global map of known services if needed and up the
ref count
@@ -360,7 +360,7 @@ public class ServiceHandler
for ( String dep : deps ) {
ServiceSet sset = serviceStateHandler.getServiceByUrl(dep);
if ( sset == null ) {
- logger.info(methodName, id, dep, "Service is unknown");
+ logger.debug(methodName, id, dep, "Service is unknown");
s.addMessage(dep, "Service is unknown.");
s.setState(ServiceState.NotAvailable);
fatal = true;
svn commit: r1863946 - in /uima/uima-ducc/trunk: uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties uima-ducc-examples/src/main/uima-ducc-vm/cron/cleanup.sh uima-ducc-sm/sr
Author: degenaro
Date: Mon Jul 29 15:15:13 2019
New Revision: 1863946
URL: http://svn.apache.org/viewvc?rev=1863946=rev
Log:
UIMA-6101 DUCC demo on Apache moved to vm3: userid duccy --> ducc
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/cleanup.sh
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties?rev=1863946=1863945=1863946=diff
==
---
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties
(original)
+++
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/config/resources/site.ducc.properties
Mon Jul 29 15:15:13 2019
@@ -45,9 +45,9 @@ ducc.broker.jmx.port=
ducc.broker.configuration = conf/activemq-ducc-unsecure.xml
# for Sun (Oracle) JVM: -XX:+HeapDumpOnOutOfMemoryError
-# for Sun (Oracle) JVM: -XX:HeapDumpPath=/home/duccy/dump
+# for Sun (Oracle) JVM: -XX:HeapDumpPath=/home/ducc/dump
-ducc.heap.dump.path = /home/duccy/dump
+ducc.heap.dump.path = /home/ducc/dump
ducc.broker.memory.options = -Xmx300M -XX:+HeapDumpOnOutOfMemoryError
ducc.agent.jvm.args= -Xmx150M -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=${ducc.heap.dump.path}
@@ -74,6 +74,6 @@ ducc.service.configuration.class=org.apa
ducc.rm.fragmentation.threshold = 1
ducc.rm.share.quantum = 15
-ducc.agent.rogue.process.user.exclusion.filter=root,postfix,ntp,nobody,daemon,100,degenaro,messagebus,nslcd,syslog,whoopsie,duccy
+ducc.agent.rogue.process.user.exclusion.filter=root,postfix,ntp,nobody,daemon,100,degenaro,messagebus,nslcd,syslog,whoopsie,ducc
ducc.agent.node.metrics.publish.rate=1
ducc.agent.node.inventory.publish.rate=5000
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/cleanup.sh
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/cleanup.sh?rev=1863946=1863945=1863946=diff
==
---
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/cleanup.sh
(original)
+++
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/cleanup.sh
Mon Jul 29 15:15:13 2019
@@ -18,21 +18,21 @@
# under the License.
# ---
-# handle /home/duccy/spool/
+# handle /home/ducc/spool/
-togo=`find /home/duccy/spool/* -mtime +21 -user duccy`
+togo=`find /home/ducc/spool/* -mtime +21 -user ducc`
echo Removing $togo
-`find /home/duccy/spool/* -mtime +21 -user duccy | xargs -i rm -rf {}`
+`find /home/ducc/spool/* -mtime +21 -user ducc | xargs -i rm -rf {}`
-# handle /home/duccy/ducc/logs/
+# handle /home/ducc/ducc/logs/
-togo=`find /home/duccy/ducc/logs/* -mtime +21 -user duccy`
+togo=`find /home/ducc/ducc/logs/* -mtime +21 -user ducc`
echo Removing $togo
-`find /home/duccy/ducc/logs/* -mtime +21 -user duccy | xargs -i rm -rf {}`
+`find /home/ducc/ducc/logs/* -mtime +21 -user ducc | xargs -i rm -rf {}`
# handle /tmp/UIMA*
-togo=`find /tmp -mtime +21 -user duccy`
+togo=`find /tmp -mtime +21 -user ducc`
echo Removing $togo
-`find /tmp -mtime +21 -user duccy | xargs -i rm -rf {}`
+`find /tmp -mtime +21 -user ducc | xargs -i rm -rf {}`
Modified:
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1863946=1863945=1863946=diff
==
---
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java
Mon Jul 29 15:15:13 2019
@@ -343,13 +343,15 @@ public class ServiceHandler
/**
* This is called when an endpoint is referenced as a dependent service
from a job or a service.
- * It is called only when a new job or service is first discovred in the
OR map.
+ * It is called only when a new job or service is first discovered in the
OR map.
*/
protected Map resolveDependencies(DuccWorkJob w,
ServiceDependency s)
{
- //String methodName = "resolveDependencies";
+ String methodName = "resolveDependencies";
DuccId id = w.getDuccId();
String[] deps = w.getServiceDependencies();
+
+logger.info(methodName, id, deps.length);
// New services
svn commit: r1863932 - /uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/boot/DuccDaemonRuntimeProperties.java
Author: degenaro
Date: Mon Jul 29 11:59:35 2019
New Revision: 1863932
URL: http://svn.apache.org/viewvc?rev=1863932=rev
Log:
UIMA-6102 DUCC daemons fail to fully boot when jmx URL is null
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/boot/DuccDaemonRuntimeProperties.java
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/boot/DuccDaemonRuntimeProperties.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/boot/DuccDaemonRuntimeProperties.java?rev=1863932=1863931=1863932=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/boot/DuccDaemonRuntimeProperties.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/boot/DuccDaemonRuntimeProperties.java
Mon Jul 29 11:59:35 2019
@@ -124,6 +124,17 @@ public class DuccDaemonRuntimeProperties
return;
}
+ private void putBootProperty(Properties bootProperties, String key,
String value) {
+ if(key != null) {
+ if(value != null) {
+ bootProperties.put(key, value);
+ }
+ else {
+ bootProperties.put(key, "?");
+ }
+ }
+ }
+
public void boot(DaemonName daemonName, String jmxUrl) {
Properties bootProperties = new Properties();
String daemonNameText = daemonName.toString();
@@ -144,12 +155,12 @@ public class DuccDaemonRuntimeProperties
catch(Exception e) {
e.printStackTrace();
}
- bootProperties.put(DuccDaemonRuntimeProperties.keyDaemonName,
daemonNameText);
- bootProperties.put(DuccDaemonRuntimeProperties.keyBootTime,
bootTime);
- bootProperties.put(DuccDaemonRuntimeProperties.keyJmxUrl,
jmxUrl);
-
bootProperties.put(DuccDaemonRuntimeProperties.keyNodeIpAddress, nodeIpAddress);
- bootProperties.put(DuccDaemonRuntimeProperties.keyNodeName,
nodeName);
- bootProperties.put(DuccDaemonRuntimeProperties.keyPid, pid);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyDaemonName, daemonNameText);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyBootTime, bootTime);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyJmxUrl, jmxUrl);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyNodeIpAddress, nodeIpAddress);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyNodeName, nodeName);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyPid, pid);
getInstance().put(daemonName, bootProperties);
}
@@ -206,12 +217,12 @@ public class DuccDaemonRuntimeProperties
catch(Exception e) {
e.printStackTrace();
}
- bootProperties.put(DuccDaemonRuntimeProperties.keyDaemonName,
name);
- bootProperties.put(DuccDaemonRuntimeProperties.keyBootTime,
bootTime);
- bootProperties.put(DuccDaemonRuntimeProperties.keyJmxUrl,
jmxUrl);
-
bootProperties.put(DuccDaemonRuntimeProperties.keyNodeIpAddress, nodeIpAddress);
- bootProperties.put(DuccDaemonRuntimeProperties.keyNodeName,
nodeName);
- bootProperties.put(DuccDaemonRuntimeProperties.keyPid, pid);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyDaemonName, name);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyBootTime, bootTime);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyJmxUrl, jmxUrl);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyNodeIpAddress, nodeIpAddress);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyNodeName, nodeName);
+ putBootProperty(bootProperties,
DuccDaemonRuntimeProperties.keyPid, pid);
getInstance().putAgent(name, bootProperties);
}
}
svn commit: r1863931 - /uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/crontab.txt
Author: degenaro Date: Mon Jul 29 11:44:33 2019 New Revision: 1863931 URL: http://svn.apache.org/viewvc?rev=1863931=rev Log: UIMA-6101 DUCC demo on Apahe moved to vm3: userid duccy --> ducc Modified: uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/crontab.txt Modified: uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/crontab.txt URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/crontab.txt?rev=1863931=1863930=1863931=diff == --- uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/crontab.txt (original) +++ uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/cron/crontab.txt Mon Jul 29 11:44:33 2019 @@ -1,2 +1,2 @@ -*/5 * * * * /home/duccy/ducc_runtime/examples/uima-ducc-vm/cron/autostart.sh -* 22 * * * /home/duccy/ducc_runtime/examples/uima-ducc-vm/cron/cleanup.sh +*/5 * * * * /home/ducc/ducc_runtime/examples/uima-ducc-vm/cron/autostart.sh +* 22 * * * /home/ducc/ducc_runtime/examples/uima-ducc-vm/cron/cleanup.sh
svn commit: r1863407 - /uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
Author: degenaro Date: Fri Jul 19 15:08:54 2019 New Revision: 1863407 URL: http://svn.apache.org/viewvc?rev=1863407=rev Log: UIMA-6086 DUCC reliability should use keepalived.conf to determine virtual ip address Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1863407=1863406=1863407=diff == --- uima/uima-ducc/trunk/src/main/resources/default.ducc.properties (original) +++ uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Fri Jul 19 15:08:54 2019 @@ -49,7 +49,8 @@ # property to the node the script is executed on. # Reliable DUCC: if running reliably, then this value must resolve to the same ip address # specified for the virtual_ipaddress in /etc/keepalived/keepalived.conf for master and -# backup nodes. DUCC CLI and Agents employ this value to connect to the current reliable +# backup nodes. To avoid nameserver glitches, consider specifying the ip address, not the +# host name. DUCC CLI and Agents employ this value to connect to the current reliable # DUCC head node. ducc.head =
svn commit: r1863406 - /uima/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/event/AgentEventListener.java
Author: degenaro
Date: Fri Jul 19 15:02:08 2019
New Revision: 1863406
URL: http://svn.apache.org/viewvc?rev=1863406=rev
Log:
UIMA-6098 DUCC reliability Agent should ignore any non-master OR publication
Modified:
uima/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/event/AgentEventListener.java
Modified:
uima/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/event/AgentEventListener.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/event/AgentEventListener.java?rev=1863406=1863405=1863406=diff
==
---
uima/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/event/AgentEventListener.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/event/AgentEventListener.java
Fri Jul 19 15:02:08 2019
@@ -18,10 +18,10 @@
*/
package org.apache.uima.ducc.agent.event;
+import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.camel.Body;
@@ -41,6 +41,7 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.ProcessStartDuccEvent;
import org.apache.uima.ducc.transport.event.ProcessStateUpdateDuccEvent;
import org.apache.uima.ducc.transport.event.ProcessStopDuccEvent;
+import org.apache.uima.ducc.transport.event.common.DuccUserReservation;
import org.apache.uima.ducc.transport.event.common.DuccWorkPopDriver;
import org.apache.uima.ducc.transport.event.common.IDuccJobDeployment;
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
@@ -237,8 +238,6 @@ public class AgentEventListener implemen
}
}
- private Map map = new ConcurrentHashMap();
-
/**
* This method is called by Camel when PM sends DUCC state to agent's
queue. It
* takes responsibility of reconciling processes on this node.
@@ -257,28 +256,34 @@ public class AgentEventListener implemen
String host = duccEvent.getProducerHost();
DuccHeadState dhs =
duccEvent.getDuccHeadState();
+ int jobs = 0;
+ List listJobs =
duccEvent.getJobList();
+ if(listJobs != null) {
+ jobs = listJobs.size();
+ }
+ int reservations = 0;
+ List listReservations =
duccEvent.getUserReservations();
+ if(listReservations != null) {
+ reservations = listReservations.size();
+ }
+ long tid = Thread.currentThread().getId();
+ String message = "sequence="+sequence+"
"+"type="+dhs+" "+"producer="+host+" "+"jobs="+jobs+"
"+"reservations="+reservations+" "+"tid="+tid;
switch(dhs) {
- case backup:
- if(!map.containsKey(host)) {
- map.put(host, host);
- logger.warn(location, jobid,
"suspended"+" "+"host:"+host);
- }
- return;
case master:
- if(map.containsKey(host)) {
- map.remove(host);
- logger.warn(location, jobid,
"resumed"+" "+"host:"+host);
- }
+ // Issue info and process master type
publication
+ logger.info(location, null,
"accept=Yes"+" "+message);
break;
default:
- break;
+ // Issue warning and ignore non-master
type publication
+ logger.warn(location, null,
"accept=No"+" "+message);
+ return;
}
// check for out of band messages. Expecting a
message with a
svn commit: r1862392 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java
Author: degenaro Date: Mon Jul 1 16:10:37 2019 New Revision: 1862392 URL: http://svn.apache.org/viewvc?rev=1862392=rev Log: UIMA-6083 DUCC BrokerHelper ERROR messages Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java?rev=1862392=1862391=1862392=diff == --- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java (original) +++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java Mon Jul 1 16:10:37 2019 @@ -38,7 +38,6 @@ import javax.management.openmbean.Compos import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccPropertiesResolver; -import org.apache.uima.ducc.common.utils.InetHelper; import org.apache.uima.ducc.common.utils.id.DuccId; import org.apache.uima.ducc.ws.DuccDaemonsData; @@ -101,7 +100,8 @@ public class BrokerHelper extends JmxHel String key; String value; // - value = InetHelper.getHostName(); + key = "ducc.broker.name"; + value = duccPropertiesResolver.getCachedProperty(key); setHost(value); // key = "ducc.broker.jmx.port";
svn commit: r1862314 - /uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Author: degenaro Date: Sat Jun 29 14:47:22 2019 New Revision: 1862314 URL: http://svn.apache.org/viewvc?rev=1862314=rev Log: UIMA-6086 DUCC reliability should use keepalived.conf to determine virtual ip address Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1862314=1862313=1862314=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Sat Jun 29 14:47:22 2019 @@ -712,21 +712,25 @@ class DuccUtil(DuccBase): label = 'get_ip_address' result = None try: -p = subprocess.Popen(['/usr/bin/nslookup', hostname], stdout=subprocess.PIPE, stderr=subprocess.PIPE) -output, err = p.communicate() -#print hostname, output, err -name = None -for line in output.splitlines(): -tokens = line.split() -if(len(tokens) == 2): -t0 = tokens[0] -t1 = tokens[1] -if(t0 == 'Address:'): -if(name != None): -result = t1 -break -elif(t0 == 'Name:'): -name = t1 +# get virtual ip address from keepalived.conf +result = self.get_virtual_ipaddress() +if(result == None): +# get virtual ip address from nameserver +p = subprocess.Popen(['/usr/bin/nslookup', hostname], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +output, err = p.communicate() +#print hostname, output, err +name = None +for line in output.splitlines(): +tokens = line.split() +if(len(tokens) == 2): +t0 = tokens[0] +t1 = tokens[1] +if(t0 == 'Address:'): +if(name != None): +result = t1 +break +elif(t0 == 'Name:'): +name = t1 except Exception as e: print e debug(label, str(result)) @@ -1355,7 +1359,7 @@ class DuccUtil(DuccBase): def get_virtual_ipaddress(self): state = 0 -vip = 'unspecified' +vip = None if ( os.path.exists(self.keepalived_conf) ): with open(self.keepalived_conf) as f: for line in f:
svn commit: r1862312 - /uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
Author: degenaro Date: Sat Jun 29 14:39:25 2019 New Revision: 1862312 URL: http://svn.apache.org/viewvc?rev=1862312=rev Log: UIMA-6086 DUCC reliability should use keepalived.conf to determine virtual ip address Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1862312=1862311=1862312=diff == --- uima/uima-ducc/trunk/src/main/resources/default.ducc.properties (original) +++ uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Sat Jun 29 14:39:25 2019 @@ -53,13 +53,6 @@ # DUCC head node. ducc.head = -# The ip address of the node where DUCC runs. -# Reliable DUCC: if running reliably, then this value must be the same ip address -# specified for the virtual_ipaddress in /etc/keepalived/keepalived.conf for master and -# backup nodes. If not specified, then the nameserver is used to get the ip address based -# on the name specified for ducc.head. -ducc.head.ip = - # Reliable DUCC: if running reliably, then this value must comprise the blank delimited list # of nodes that are eligible to become the DUCC head node. Admin commands start_ducc and # stop_ducc are only allowed on the ducc.head node or any node in the ducc.head.reliable.list.
svn propchange: r1862310 - svn:log
Author: degenaro Revision: 1862310 Modified property: svn:log Modified: svn:log at Sat Jun 29 14:21:06 2019 -- --- svn:log (original) +++ svn:log Sat Jun 29 14:21:06 2019 @@ -0,0 +1 @@ +UIMA-6086 DUCC reliability should use keepalived.conf to determine virtual ip address
svn commit: r1862310 - in /uima/uima-ducc/trunk/src/main: admin/ducc_util.py admin/virtual_ipaddress.py resources/default.ducc.properties
Author: degenaro
Date: Sat Jun 29 14:04:20 2019
New Revision: 1862310
URL: http://svn.apache.org/viewvc?rev=1862310=rev
Log: (empty)
Added:
uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py (with props)
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1862310=1862309=1862310=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Sat Jun 29 14:04:20 2019
@@ -1351,13 +1351,49 @@ class DuccUtil(DuccBase):
return False
return True
-keepalivd_conf = '/etc/keepalived/keepalived.conf'
+keepalived_conf = '/etc/keepalived/keepalived.conf'
+
+def get_virtual_ipaddress(self):
+state = 0
+vip = 'unspecified'
+if ( os.path.exists(self.keepalived_conf) ):
+with open(self.keepalived_conf) as f:
+for line in f:
+tokens = line.split(' ')
+if(tokens == None):
+pass
+elif(len(tokens) == 0):
+pass
+elif(tokens[0] == '!'):
+pass
+else:
+for token in tokens:
+token = token.strip()
+if(len(token) == 0):
+continue
+elif(token == '#'):
+break
+if(state == 0):
+if(token == 'virtual_ipaddress'):
+state = 1
+elif(state == 1):
+if(token == '{'):
+state = 2
+elif(state == 2):
+vip = token
+state = 3
+elif(state == 3):
+if(token == '}'):
+state = 4
+else:
+pass
+return vip
# eligible when keepalived config comprises the ip
def is_reliable_eligible(self, ip):
retVal = False
-if ( os.path.exists(self.keepalivd_conf) ):
-with open(self.keepalivd_conf) as f:
+if ( os.path.exists(self.keepalived_conf) ):
+with open(self.keepalived_conf) as f:
for line in f:
if ip in line:
retVal = True
Added: uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py?rev=1862310=auto
==
--- uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py (added)
+++ uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py Sat Jun 29
14:04:20 2019
@@ -0,0 +1,31 @@
+#! /usr/bin/env python
+# ---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ---
+
+from ducc_util import DuccUtil
+
+class VirtualIpAddress(DuccUtil):
+
+def main(self):
+ result = self.get_virtual_ipaddress()
+ print result
+
+if __name__ == '__main__':
+instance = VirtualIpAddress()
+instance.main()
\ No newline at end of file
Propchange: uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py
--
svn:executable = *
Propchange: uima/uima-ducc/trunk/src/main/admin/virtual_ipaddress.py
--
svn:mime-type = text/plain
Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.p
svn commit: r1862087 - in /uima/uima-ducc/trunk: uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/Jo
Author: degenaro
Date: Tue Jun 25 20:59:44 2019
New Revision: 1862087
URL: http://svn.apache.org/viewvc?rev=1862087=rev
Log:
UIMA-6077 DUCC stop_ducc -ag should cause RM to stop scheduling there
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java?rev=1862087=1862086=1862087=diff
==
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Tue Jun 25 20:59:44 2019
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.camel.CamelContext;
@@ -725,6 +726,42 @@ implements Orchestrator {
return dh;
}
+ private void diagnose(DuccHeadTransition dh_transition, String when) {
+ String methodName = "diagnose";
+ DuccWorkMap dwm = orchestratorCommonArea.getWorkMap();
+ Set keys = dwm.getJobKeySet();
+ if(keys.isEmpty()) {
+ logger.debug(methodName, null, "no jobs");
+ }
+ else {
+ for(DuccId key : keys) {
+ IDuccWork dw = dwm.findDuccWork(key);
+ IDuccWorkJob dwj = (IDuccWorkJob) dw;
+ if(dwj != null) {
+ IDuccProcessMap processMap =
dwj.getProcessMap();
+ if(processMap != null) {
+ for(IDuccProcess process :
processMap.values()) {
+ StringBuffer sb = new
StringBuffer();
+
sb.append("head-state:"+dh_transition.name()+"="+when);
+ sb.append(" ");
+
sb.append("ducc-pid:"+process.getDuccId());
+ sb.append(" ");
+
sb.append("user:"+dwj.getStandardInfo().getUser());
+ sb.append(" ");
+
sb.append("node:"+process.getNode().getNodeIdentity().getCanonicalName());
+ sb.append(" ");
+
sb.append("pid:"+process.getPID());
+
logger.debug(methodName, dwj.getDuccId(), sb);
+ }
+ }
+ }
+ else {
+ logger.debug(methodName, key, "no job");
+ }
+ }
+ }
+ }
+
/**
* Publish Orchestrator State
*/
@@ -745,10 +782,12 @@ implements Orchestrator {
logger.warn(methodName, jobid, "ducc head -> backup");
break;
case backup_to_master:
+ diagnose(dh_transition,"before");
OrchestratorCommonArea.getInstance().restart();
SystemEventsLogger.warn(IDuccLoggerComponents.abbrv_orchestrator,
EventType.SWITCH_TO_MASTER.name(), "");
orchestratorStateDuccEvent.setDuccHeadState(DuccHeadState.master);
logger.warn(methodName, jobid, "ducc head -> master");
+ diagnose(dh_transition,"after");
break;
case master_to_master:
orchestratorStateDuccEvent.setDuccHeadState(DuccHeadState.master);
Modified:
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/
svn commit: r1862012 - /uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverStateExchanger.java
Author: degenaro
Date: Mon Jun 24 15:36:24 2019
New Revision: 1862012
URL: http://svn.apache.org/viewvc?rev=1862012=rev
Log:
UIMA-6080 DUCC Job Driver (JD) NPE doing status exchange with Orchestrator (OR)
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverStateExchanger.java
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverStateExchanger.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverStateExchanger.java?rev=1862012=1862011=1862012=diff
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverStateExchanger.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverStateExchanger.java
Mon Jun 24 15:36:24 2019
@@ -180,24 +180,17 @@ public class JobDriverStateExchanger ext
private void abortIfTold(JdReplyEvent jdReplyEvent) {
String location = "abortIfTold";
- if(jdReplyEvent != null) {
- if(jdReplyEvent.isDuccHeadMaster()) {
- String killDriverReason =
jdReplyEvent.getKillDriverReason();
- if(killDriverReason != null) {
- int code = 255;
- StringBuffer sb = new StringBuffer();
- sb.append("System Exit");
- sb.append(" ");
- sb.append("code="+code);
- sb.append(" ");
- sb.append("reason="+killDriverReason);
- logger.warn(location, jobid,
sb.toString());
- System.exit(code);
- }
- }
- else {
- logger.warn(location, jobid, "not master");
- }
+ String killDriverReason = jdReplyEvent.getKillDriverReason();
+ if(killDriverReason != null) {
+ int code = 255;
+ StringBuffer sb = new StringBuffer();
+ sb.append("System Exit");
+ sb.append(" ");
+ sb.append("code="+code);
+ sb.append(" ");
+ sb.append("reason="+killDriverReason);
+ logger.warn(location, jobid, sb.toString());
+ System.exit(code);
}
}
@@ -316,7 +309,22 @@ public class JobDriverStateExchanger ext
try {
JdRequestEvent jdRequestEvent = getJdRequestEvent();
JdReplyEvent jdReplyEvent = request(jdRequestEvent);
- handle(jdReplyEvent);
+ if(jdReplyEvent != null) {
+ if(jdReplyEvent.isDuccHeadMaster()) {
+ if(jdReplyEvent.getProcessMap() !=
null) {
+ handle(jdReplyEvent);
+ }
+ else {
+ logger.warn(location, jobid,
"no map");
+ }
+ }
+ else {
+ logger.warn(location, jobid, "not
master");
+ }
+ }
+ else {
+ logger.warn(location, jobid, "null reply");
+ }
if(!communications_ok) {
logger.warn(location, jobid, "Status reporting
resumed.");
communications_ok = true;
svn commit: r1861847 - /uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Author: degenaro
Date: Sat Jun 22 11:45:17 2019
New Revision: 1861847
URL: http://svn.apache.org/viewvc?rev=1861847=rev
Log:
UIMA-6078 DUCC Orchestrator (OR) too verbose...
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java?rev=1861847=1861846=1861847=diff
==
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorComponent.java
Sat Jun 22 11:45:17 2019
@@ -462,14 +462,14 @@ implements Orchestrator {
boolean is_master = getDuccHead().is_ducc_head_virtual_master();
if(!is_master) {
duccEvent.setDuccHeadMaster(false);
- logger.info(methodName, duccId, "not master");
+ logger.debug(methodName, duccId, "not master");
}
else {
duccEvent.setDuccHeadMaster(true);
- logger.info(methodName, duccId, "master");
+ logger.debug(methodName, duccId, "master");
DuccWorkMap workMap =
orchestratorCommonArea.getWorkMap();
if(dsr != null) {
- logger.info(methodName, duccId,
dsr.getLogReport());
+ logger.debug(methodName, duccId,
dsr.getLogReport());
stateManager.reconcileState(dsr);
String sid = ""+duccId.getFriendly();
DuccWorkJob duccWorkJob = (DuccWorkJob)
WorkMapHelper.cloneDuccWork(workMap, sid, this, methodName);
svn commit: r1861788 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java
Author: degenaro
Date: Fri Jun 21 15:01:15 2019
New Revision: 1861788
URL: http://svn.apache.org/viewvc?rev=1861788=rev
Log:
UIMA-6072 DUCC Web Server (WS) authentication check for user = self broken
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java?rev=1861788=1861787=1861788=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java
Fri Jun 21 15:01:15 2019
@@ -426,12 +426,20 @@ public class HandlersHelper {
String methodName = "getServiceAuthorization";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
DataAccessPermission retVal = DataAccessPermission.None;
+
try {
- if(resOwner == reqUser) {
+ if(resOwner == null) {
+ // None
+ }
+ else if(reqUser == null) {
+ // None
+ }
+ else if(resOwner.equals(reqUser)) {
retVal = DataAccessPermission.Read;
}
else {
String home = getSecurityHome(resOwner.trim());
+ duccLogger.debug(methodName, jobid,
"owner="+resOwner+" "+"home"+home);
if(home != null) {
if(!home.endsWith(File.separator)) {
home = home+File.separator;
@@ -440,10 +448,10 @@ public class HandlersHelper {
boolean readable =
isFileReadable(reqUser, path);
if(readable) {
retVal =
DataAccessPermission.Read;
- duccLogger.debug(methodName,
jobid, "owner="+resOwner+" "+"user="+reqUser+" "+retVal);
}
}
}
+ duccLogger.debug(methodName, jobid, "owner="+resOwner+"
"+"user="+reqUser+" "+retVal);
}
catch(Exception e) {
duccLogger.error(methodName, jobid, e);
svn commit: r1861708 - /uima/uima-ducc/trunk/src/main/admin/ducc_rsync
Author: degenaro Date: Thu Jun 20 18:17:20 2019 New Revision: 1861708 URL: http://svn.apache.org/viewvc?rev=1861708=rev Log: UIMA-6031 ducc_rsync options could be more useful - modify default file sets for agent and head Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1861708=1861707=1861708=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Thu Jun 20 18:17:20 2019 @@ -59,18 +59,19 @@ class DuccRsync(DuccUtil): 'duccling', 'examples', #'history', -#'INSTALL', -#'issuesFixed', +'INSTALL', +'issuesFixed', 'lib', -#'LICENSE', +'LICENSE', #'logs', -#'NOTICE', -#'README', -#'RELEASE_NOTES.html', -'webserver', +'NOTICE', +'README', +'RELEASE_NOTES.html', 'resources', 'resources.private', #'state', +'state/duccling.version', +'webserver', ] agent_dirs_list = [ @@ -83,18 +84,19 @@ class DuccRsync(DuccUtil): 'duccling', 'examples', #'history', -#'INSTALL', -#'issuesFixed', +'INSTALL', +'issuesFixed', 'lib', -#'LICENSE', +'LICENSE', #'logs', -#'NOTICE', -#'README', -#'RELEASE_NOTES.html', -#'webserver', +'NOTICE', +'README', +'RELEASE_NOTES.html', 'resources', -'resources.private', +'resources.private/ducc-broker-credentials.properties', #'state', +'state/duccling.version', +#'webserver', ] dual_dirs_list = head_dirs_list @@ -122,8 +124,8 @@ class DuccRsync(DuccUtil): self.help_head_nodes = 'Blank separated list of head nodes to synchronize, default:"None"; specify "all" for '+' '.join(self.head_nodes) self.help_agent_nodes = 'Blank separated list of agent nodes to synchronize, default:"None"; specify "all" for '+' '.join(self.agent_nodes) -self.help_head_dirs = 'Blank separated list of head directories to synchronize, default="'+self.head_dirs+'"' -self.help_agent_dirs = 'Blank separated list of agent directories to synchronize, default="'+self.agent_dirs+'"' +self.help_head_dirs = 'Blank separated list of head directories (and files) to synchronize, default="'+self.head_dirs+'"' +self.help_agent_dirs = 'Blank separated list of agent directories (and files) to synchronize, default="'+self.agent_dirs+'"' self.help_debug = 'Display debugging messages.' self.help_quiet = 'Suppress informational messages.'
svn commit: r1861688 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server: DuccAbstractHandler.java DuccHandlerClassic.java
Author: degenaro
Date: Thu Jun 20 13:04:49 2019
New Revision: 1861688
URL: http://svn.apache.org/viewvc?rev=1861688=rev
Log:
UIMA-6071 DUCC cli ducc_status info is deficient
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java?rev=1861688=1861687=1861688=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
Thu Jun 20 13:04:49 2019
@@ -185,6 +185,16 @@ public abstract class DuccAbstractHandle
private DateFormat format = new SimpleDateFormat(".MM.dd HH:mm:ss
E", Locale.ENGLISH);
+ public String toDash(String value) {
+ String retVal = "-";
+ if(value != null) {
+ if(value.length() > 0) {
+ retVal = value;
+ }
+ }
+ return retVal;
+ }
+
public String getTimeStamp(DateStyle dateStyle, long tod) {
String methodName = "";
Date date = new Date(tod);
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1861688=1861687=1861688=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
Thu Jun 20 13:04:49 2019
@@ -1611,11 +1611,11 @@ public class DuccHandlerClassic extends
sb.append("");
// Boot Time
sb.append("");
- sb.append(bootTime);
+ sb.append(toDash(bootTime));
sb.append("");
// Host IP
sb.append("");
- sb.append(hostIP);
+ sb.append(toDash(hostIP));
sb.append("");
// Host Name
sb.append("");
@@ -1623,7 +1623,7 @@ public class DuccHandlerClassic extends
sb.append("");
// PID
sb.append("");
- sb.append(pid);
+ sb.append(toDash(pid));
sb.append("");
// Publication Size (last)
sb.append("");
svn commit: r1860631 - /uima/uima-ducc/trunk/src/main/admin/ducc_rsync
Author: degenaro
Date: Tue Jun 4 19:28:59 2019
New Revision: 1860631
URL: http://svn.apache.org/viewvc?rev=1860631=rev
Log:
UIMA-6031 ducc_rsync options could be more useful
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_rsync
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1860631=1860630=1860631=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Tue Jun 4 19:28:59 2019
@@ -49,6 +49,60 @@ class DuccRsync(DuccUtil):
rsync_cmd = 'rsync'
rsync_flags = '-avz --delete --ignore-errors'
+head_dirs_list = [
+'admin',
+'apache-uima',
+'bin',
+'cassandra-server',
+'cron',
+'docs',
+'duccling',
+'examples',
+#'history',
+#'INSTALL',
+#'issuesFixed',
+'lib',
+#'LICENSE',
+#'logs',
+#'NOTICE',
+#'README',
+#'RELEASE_NOTES.html',
+'webserver',
+'resources',
+'resources.private',
+#'state',
+]
+
+agent_dirs_list = [
+'admin',
+'apache-uima',
+'bin',
+#'cassandra-server',
+'cron',
+'docs',
+'duccling',
+'examples',
+#'history',
+#'INSTALL',
+#'issuesFixed',
+'lib',
+#'LICENSE',
+#'logs',
+#'NOTICE',
+#'README',
+#'RELEASE_NOTES.html',
+#'webserver',
+'resources',
+'resources.private',
+#'state',
+]
+
+dual_dirs_list = head_dirs_list
+
+head_dirs = ' '.join(head_dirs_list)
+agent_dirs = ' '.join(agent_dirs_list)
+dual_dirs = ' '.join(dual_dirs_list)
+
def __init__(self):
DuccUtil.__init__(self, self.merge)
@@ -61,29 +115,35 @@ class DuccRsync(DuccUtil):
def get_epilog(self):
epilog = ''
epilog = epilog+'Purpose: synchronize DUCC directory(s) from present
node to other head & agent nodes.'
-epilog = epilog+'\n\n'
-epilog = epilog+'Example:'
-epilog = epilog+'\n\n'
-epilog = epilog+'> '+self._fn()+' '+'--dirs "resources admin"
--nodelist ducc.nodes'
return epilog
-
-default_dirs = 'resources'
-default_nodelist = 'ducc.nodes'
-
-help_dirs= 'Blank separated list of directories to synchronize,
default='+default_dirs+'.'
-help_nodelist= 'Blank separated list of file names,
default='+default_nodelist+'.'
-help_debug = 'Display debugging messages.'
-help_quiet = 'Suppress informational messages.'
# parse command line
def get_args(self):
+self.help_head_nodes = 'Blank separated list of head nodes to
synchronize, default:"None"; specify "all" for '+' '.join(self.head_nodes)
+self.help_agent_nodes = 'Blank separated list of agent nodes to
synchronize, default:"None"; specify "all" for '+' '.join(self.agent_nodes)
+
+self.help_head_dirs = 'Blank separated list of head directories to
synchronize, default="'+self.head_dirs+'"'
+self.help_agent_dirs = 'Blank separated list of agent directories to
synchronize, default="'+self.agent_dirs+'"'
+
+self.help_debug = 'Display debugging messages.'
+self.help_quiet = 'Suppress informational messages.'
+
self.parser =
argparse.ArgumentParser(formatter_class=RawDescriptionHelpFormatter,epilog=self.get_epilog())
-self.parser.add_argument('--dirs' , '-d', action='store',
default=self.default_dirs, help=self.help_dirs)
-self.parser.add_argument('--nodelist' , '-n', action='store',
default=self.default_nodelist, help=self.help_nodelist)
-self.parser.add_argument('--debug', '-g', action='store_true',
help=self.help_debug)
-self.parser.add_argument('--quiet', '-q', action='store_true',
help=self.help_quiet)
+
+self.parser.add_argument('--head-nodes' , action='store' ,
help=self.help_head_nodes)
+self.parser.add_argument('--agent-nodes' , action='store' ,
help=self.help_agent_nodes)
+self.parser.add_argument('--head-dirs' , action='store' ,
help=self.help_head_dirs)
+self.parser.add_argument('--agent-dirs' , action='store' ,
help=self.help_agent_dirs)
+
+self.parser.add_argument('--debug' , action='store_true',
help=self.help_debug)
+self.parser.add_argument('--quiet' , action='store_true',
help=self.help_quiet)
+
self.args = self.parser.parse_args()
-
+
+if(self.args.head_nodes == None):
+if(self.args.agent_nodes == None):
+
svn commit: r1860332 - /uima/uima-ducc/trunk/src/main/scripts/properties.py
Author: degenaro
Date: Wed May 29 11:58:16 2019
New Revision: 1860332
URL: http://svn.apache.org/viewvc?rev=1860332=rev
Log:
UIMA-6003 DUCC start_sim fails to create a pids file
Modified:
uima/uima-ducc/trunk/src/main/scripts/properties.py
Modified: uima/uima-ducc/trunk/src/main/scripts/properties.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/scripts/properties.py?rev=1860332=1860331=1860332=diff
==
--- uima/uima-ducc/trunk/src/main/scripts/properties.py (original)
+++ uima/uima-ducc/trunk/src/main/scripts/properties.py Wed May 29 11:58:16 2019
@@ -219,7 +219,7 @@ class Properties:
#
def _makedirs(self, path):
try:
-dir_path = path.rsplit('/',1)[0]
+dir_path = os.path.dirpath(path)
os.makedirs(dir_path)
except:
pass
svn commit: r1859673 - in /uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state: DuccWorkIdFactory.java OrchestratorStateDb.java
Author: degenaro
Date: Tue May 21 19:23:13 2019
New Revision: 1859673
URL: http://svn.apache.org/viewvc?rev=1859673=rev
Log:
UIMA-6049 DUCC Orchestrator (OR) may give out duplicate ID numbers
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/DuccWorkIdFactory.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/OrchestratorStateDb.java
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/DuccWorkIdFactory.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/DuccWorkIdFactory.java?rev=1859673=1859672=1859673=diff
==
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/DuccWorkIdFactory.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/DuccWorkIdFactory.java
Tue May 21 19:23:13 2019
@@ -27,7 +27,7 @@ public class DuccWorkIdFactory implement
private static IOrchestratorState orchestratorState =
OrchestratorState.getInstance();
@Override
- public DuccId next() {
+ public synchronized DuccId next() {
long value = orchestratorState.getNextDuccWorkSequenceNumber();
return new DuccId(value);
}
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/OrchestratorStateDb.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/OrchestratorStateDb.java?rev=1859673=1859672=1859673=diff
==
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/OrchestratorStateDb.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/state/OrchestratorStateDb.java
Tue May 21 19:23:13 2019
@@ -76,7 +76,7 @@ public class OrchestratorStateDb impleme
}
@Override
- public long getNextDuccWorkSequenceNumber() {
+ public synchronized long getNextDuccWorkSequenceNumber() {
long prev = orDbOrchestratorProperties.getDuccWorkSeqNo();
long next = prev+1;
orDbOrchestratorProperties.setDuccWorkSeqNo(next);
svn commit: r1859428 - in /uima/uima-ducc/trunk/src/main/admin: check_ducc ducc_rsync
Author: degenaro Date: Fri May 17 12:21:56 2019 New Revision: 1859428 URL: http://svn.apache.org/viewvc?rev=1859428=rev Log: UIMA-6035 ducc.properties not updated with site.ducc.properties when needed Modified: uima/uima-ducc/trunk/src/main/admin/check_ducc uima/uima-ducc/trunk/src/main/admin/ducc_rsync Modified: uima/uima-ducc/trunk/src/main/admin/check_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/check_ducc?rev=1859428=1859427=1859428=diff == --- uima/uima-ducc/trunk/src/main/admin/check_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/check_ducc Fri May 17 12:21:56 2019 @@ -36,7 +36,7 @@ from ducc_util import ThreadPool class CheckDucc(DuccUtil): def __init__(self): -DuccUtil.__init__(self) +DuccUtil.__init__(self,merge=True) self.badnodes = [] def validate(self, checkdate): Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1859428=1859427=1859428=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Fri May 17 12:21:56 2019 @@ -41,7 +41,7 @@ from ducc_util import * class DuccRsync(DuccUtil): -merge = False +merge = True key_ducc_head = 'ducc.head' key_ducc_head_reliable_list = 'ducc.head.reliable.list'
svn commit: r1859329 - in /uima/uima-ducc/trunk: src/main/admin/ducc_util.py uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java uima-ducc-database/src/main/java/org/apache/uim
Author: degenaro
Date: Wed May 15 20:15:40 2019
New Revision: 1859329
URL: http://svn.apache.org/viewvc?rev=1859329=rev
Log:
UIMA-6027 org.apache.uima.ducc.database.DbAlive improvement
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1859329=1859328=1859329=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Wed May 15 20:15:40 2019
@@ -338,7 +338,7 @@ class DuccUtil(DuccBase):
dbnode = self.get_db_host()
-CMD = [self.java(), 'org.apache.uima.ducc.database.DbAlive', dbnode,
'ducc', self.db_password, str(retry)]
+CMD = [self.java(), 'org.apache.uima.ducc.database.DbAlive',
self.DUCC_HOME, dbnode, str(retry)]
CMD = ' '.join(CMD)
if(not verbose):
Modified:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java?rev=1859329=1859328=1859329=diff
==
---
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbAlive.java
Wed May 15 20:15:40 2019
@@ -33,8 +33,6 @@ import com.datastax.driver.core.exceptio
public class DbAlive
{
String db_host_list;
-String adminid = null;
-String adminpw = null;
enum RC {
OK {
@@ -65,14 +63,12 @@ public class DbAlive
private Cluster cluster;
//private Session session;
-public DbAlive(String db_host_list, String adminid, String adminpw)
+public DbAlive(String db_host_list)
{
this.db_host_list = db_host_list;
-this.adminid = adminid;
-this.adminpw = adminpw;
}
-public RC connect()
+private RC connect(String adminid, String adminpw)
throws Exception
{
RC ret = RC.OK;
@@ -124,19 +120,19 @@ public class DbAlive
static void usage()
{
-System.out.println("Usage: DbAlive database_url id pw retry-count");
+System.out.println("Usage: DbAlive ducc_home database_url
retry_count");
System.exit(1);
}
public static void main(String[] args)
{
-if ( args.length != 4 ) {
+if ( args.length != 3 ) {
usage();
}
int max = 0;
try {
-max = Integer.parseInt(args[3]); // we'll
wait up to 60 seconds: 20 x 3 seconds
+max = Integer.parseInt(args[2]); // we'll
wait up to 60 seconds: 20 x 3 seconds
} catch ( NumberFormatException e ) {
System.out.println("Retry count must be numeric.");
System.exit(1);
@@ -145,9 +141,12 @@ public class DbAlive
DbAlive client = null;
RC rc = RC.OK;
try {
-client = new DbAlive(args[0], args[1], args[2]);
+ String ducc_home = args[0];
+ String adminid = DbManager.getDbUser(ducc_home);
+ String adminpw = DbManager.getDbPassword(ducc_home);
+client = new DbAlive(args[1]);
for ( int i = 0; i < max; i++ ) {
-rc = client.connect();
+rc = client.connect(adminid, adminpw);
System.out.println(rc.message());
if ( rc == RC.OK) {
break;
Modified:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1859329=1859328=1859329=diff
==
---
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
Wed May 15 20:15:40 2019
@@ -164,25 +164,41 @@ public class DbManager
return session.execute(s);
}
-static void dbPassword()
- throws Exception
+private static void dbPassword() throws Exception
{
-File f = new File(System.getProperty("DUCC_HOME") +
"/resources.private/" + DbCreate.PASSWORD_FILE);
-
svn commit: r1859242 - in /uima/uima-ducc/trunk: src/main/admin/ uima-ducc-database/src/main/java/org/apache/uima/ducc/database/
Author: degenaro Date: Tue May 14 18:15:36 2019 New Revision: 1859242 URL: http://svn.apache.org/viewvc?rev=1859242=rev Log: UIMA-6033 quiesced column not created when extending from older DUCC DB restore. Added: uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUpdate.java (with props) Modified: uima/uima-ducc/trunk/src/main/admin/db_util.py uima/uima-ducc/trunk/src/main/admin/start_ducc uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java Modified: uima/uima-ducc/trunk/src/main/admin/db_util.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_util.py?rev=1859242=1859241=1859242=diff == --- uima/uima-ducc/trunk/src/main/admin/db_util.py (original) +++ uima/uima-ducc/trunk/src/main/admin/db_util.py Tue May 14 18:15:36 2019 @@ -205,3 +205,12 @@ def configure_database(DUCC_HOME, DUCC_H if( db_autostart ): stop_database(pidfile) return ret + +def update_database(DUCC_HOME, jvm): +CMD = [jvm, '-DDUCC_HOME=' + DUCC_HOME, 'org.apache.uima.ducc.database.DbUpdate'] +CMD = ' '.join(CMD) +if ( execute(CMD) != 0 ): +print 'Database schema update failure.' +#else: +#print 'Database schema update success.' + Modified: uima/uima-ducc/trunk/src/main/admin/start_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/start_ducc?rev=1859242=1859241=1859242=diff == --- uima/uima-ducc/trunk/src/main/admin/start_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/start_ducc Tue May 14 18:15:36 2019 @@ -34,6 +34,8 @@ from duccimport Ducc from ducc_util import ThreadPool from ducc_base import find_ducc_home +import db_util as dbu + class StartDucc(DuccUtil): def __init__(self): @@ -369,6 +371,7 @@ class StartDucc(DuccUtil): node = self.get_db_host() com = 'database' self.db_acct_start(node,com) +dbu.update_database(self.DUCC_HOME, self.jvm) except Exception (e): # print e print sys.exc_info()[0], "Can't start the database." Added: uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUpdate.java URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUpdate.java?rev=1859242=auto == --- uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUpdate.java (added) +++ uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUpdate.java Tue May 14 18:15:36 2019 @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +*/ + +package org.apache.uima.ducc.database; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.apache.uima.ducc.common.utils.DuccLogger; + +import com.datastax.driver.core.AuthProvider; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.PlainTextAuthProvider; +import com.datastax.driver.core.Session; +import com.datastax.driver.core.SimpleStatement; +import com.datastax.driver.core.exceptions.InvalidQueryException; + +/** + * The purpose of this class is to update the DB with additions for newer versions of DUCC. + * + * For example, when upgrading from DUCC 2.2.2 to 3.0.0 the new column "quiesced" needs to + * be added to the rmnodes table. The methodology is to call the RmStatePersistence class to + * get an alter statement, and execute it against the "old" database in order to modernize it. + * If installing 3.0.0, the column is already there. Adding the column more than once is harmless. + *
svn commit: r1859098 - /uima/uima-ducc/trunk/src/main/admin/db_create
Author: degenaro Date: Fri May 10 18:23:17 2019 New Revision: 1859098 URL: http://svn.apache.org/viewvc?rev=1859098=rev Log: UIMA-6033 quesced column not created when extending from older DUCC DB restore. Modified: uima/uima-ducc/trunk/src/main/admin/db_create Modified: uima/uima-ducc/trunk/src/main/admin/db_create URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_create?rev=1859098=1859097=1859098=diff == --- uima/uima-ducc/trunk/src/main/admin/db_create (original) +++ uima/uima-ducc/trunk/src/main/admin/db_create Fri May 10 18:23:17 2019 @@ -71,7 +71,9 @@ class DbCreate(DuccUtil): sys.exit(1) def main(self, argv): - + +self.cmd_name_nodetool = 'nodetool' + self.database_automanage = True self.database_host_list = None self.database_user = None @@ -172,5 +174,5 @@ class DbCreate(DuccUtil): if __name__ == "__main__": -postinstall = DbCreate() -postinstall.main(sys.argv[1:]) +instance = DbCreate() +instance.main(sys.argv[1:])
svn commit: r1858583 - /uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
Author: degenaro
Date: Fri May 3 20:22:54 2019
New Revision: 1858583
URL: http://svn.apache.org/viewvc?rev=1858583=rev
Log:
UIMA-6030 db_tool bad documentation
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex?rev=1858583=1858582=1858583=diff
==
---
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
(original)
+++
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/admin-commands.tex
Fri May 3 20:22:54 2019
@@ -436,7 +436,7 @@ check_ducc -n nlist1 -n nlist2
\label{subsec:admin.ducc-update}
\paragraph{Description:}
-This command is used to unpack a new release of DUCC and create a new
installion or update
+This command is used to unpack a new release of DUCC and create a new
installation or update
an existing one.
For a new installation it simply unpacks the tar file with the
appropriate permissions.
The setup must be completed by running
\hyperref[subsec:admin.ducc-post-install]{\em ducc\_post\_install}.
@@ -644,7 +644,7 @@ Nodepool power
\begin{itemize}
\item The name of the host,
\item Whether the host has any blacklists processes on it,
- \item Whether the host is currently onlline (responsive),
+ \item Whether the host is currently online (responsive),
\item The status of the host; whether the host is schedulable ({\em up}
or {\em down}. A responsive host becomes
unschedulable ({\em down}) if it is varied-off,
\item The nodepool the host is a member of,
@@ -652,7 +652,7 @@ Nodepool power
\item The {\em order} of the host. The {\em order} is defined to be the
maximum number of quantum shares
supported by the host,
\item The number of unscheduled quantum shares on the host, and
- \item If work is scheduled on the host, information relevent to that
scheduled processes (or reservation).
+ \item If work is scheduled on the host, information relevant to that
scheduled processes (or reservation).
\end{itemize}
If work is scheduled on a host, the work summary is keyed thus:
@@ -669,7 +669,7 @@ Nodepool power
\item[E] Whether the RM has preempted (evicted) the process but it has
not yet exited,
\item[P] Whether the RM has purged the process (evicted, because the
host is non-responsive), but it has not
been confirmed evicted,
- \item[F] Whether the process is {\em fixed}; that is, non-preemptbable,
+ \item[F] Whether the process is {\em fixed}; that is, non-preemptable,
\item[I] Whether the initialization phase is completed (usually only
UIMA-AS processes).
\end{description}
@@ -708,7 +708,7 @@ Nodepool power
Vary\_off is used to remove a host from scheduling and to evict the
preemptable work that is running on it.
This allows for graceful clearance of a host so the host can be take
offline for maintenance,
or any other purpose (such as sharing the host with other applications.)
-The DUCC agent is NOT stoppped; use
\hyperref[subsec:admin.stop-ducc]{stop\_ducc} to stop the
+The DUCC agent is NOT stopped; use
\hyperref[subsec:admin.stop-ducc]{stop\_ducc} to stop the
agent.
Managed and unmanaged reservations are not canceled by {\em vary\_off}.
@@ -840,8 +840,8 @@ Nodepool power
\item Loads all reservation history from {\em
DUCC\_HOME/history/reservations}.
\item Loads all service instance and AP history from {\em
DUCC\_HOME/history/services}.
\item Loads the service registry from {\em
DUCC\_HOME/state/services}.
- \item Loads the service registry histroy from {\em
DUCC\_HOME/history/service-registry}.
- \item Reloads the Orchestratory checkpoint, as a spot-check of the
loader's instrumentation (to insure
+ \item Loads the service registry history from {\em
DUCC\_HOME/history/service-registry}.
+ \item Reloads the Orchestrator checkpoint, as a spot-check of the
loader's instrumentation (to insure
load times stay reasonable.)
\item Re-installs the DUCC database schema.
\item Stops the database.
@@ -892,20 +892,20 @@ db_loader -i /home/ducc.old/ducc_runtime
\paragraph{Options:}
Exactly one of the following options is required.
\begin{description}
- \item[$--save$ {\em PATH.tar.gz}]
+ \item[$--$save {\em PATH.tar.gz}]
The file to save the DUCC database contents. The database
must be up. The file cannot already exist.
- \item[$--save
svn propchange: r1838023 - svn:log
Author: degenaro Revision: 1838023 Modified property: svn:log Modified: svn:log at Wed May 1 18:29:56 2019 -- --- svn:log (original) +++ svn:log Wed May 1 18:29:56 2019 @@ -0,0 +1 @@ +CVE-2018-8035: Apache UIMA DUCC webserver cross-site scripting (XSS) vulnerability due to unintended execution of user supplied javascript code.
svn commit: r1858317 - in /uima/site/trunk/uima-website: docs/ xdocs/stylesheets/
Author: degenaro Date: Sun Apr 28 11:18:37 2019 New Revision: 1858317 URL: http://svn.apache.org/viewvc?rev=1858317=rev Log: UIMA-6028 Add "Doing a CVE" link to Development section Modified: uima/site/trunk/uima-website/docs/annotators.html uima/site/trunk/uima-website/docs/apache-board-status.html uima/site/trunk/uima-website/docs/building-uima.html uima/site/trunk/uima-website/docs/bulk-contribution-checklist.html uima/site/trunk/uima-website/docs/checklist-release.html uima/site/trunk/uima-website/docs/code-scan-tools.html uima/site/trunk/uima-website/docs/codeConventions.html uima/site/trunk/uima-website/docs/coling14.html uima/site/trunk/uima-website/docs/communication.html uima/site/trunk/uima-website/docs/contribution-policy.html uima/site/trunk/uima-website/docs/decisions.html uima/site/trunk/uima-website/docs/dependencies.html uima/site/trunk/uima-website/docs/dev-docbook.html uima/site/trunk/uima-website/docs/dev-eclipse-plugin-archiving.html uima/site/trunk/uima-website/docs/dev-eclipse-plugin-signing.html uima/site/trunk/uima-website/docs/dev-quick.html uima/site/trunk/uima-website/docs/distribution-before-2.3.1.html uima/site/trunk/uima-website/docs/distribution.html uima/site/trunk/uima-website/docs/doc-uima-annotator.html uima/site/trunk/uima-website/docs/doc-uima-examples.html uima/site/trunk/uima-website/docs/doc-uima-pears.html uima/site/trunk/uima-website/docs/doc-uima-v3-samples.html uima/site/trunk/uima-website/docs/doc-uima-why.html uima/site/trunk/uima-website/docs/doc-uimaas-what.html uima/site/trunk/uima-website/docs/doc-uimacpp-huh.html uima/site/trunk/uima-website/docs/doc-uimaducc-demo.html uima/site/trunk/uima-website/docs/doc-uimaducc-whatitam.html uima/site/trunk/uima-website/docs/documentation.html uima/site/trunk/uima-website/docs/downloads.html uima/site/trunk/uima-website/docs/eclipse-update-archives.html uima/site/trunk/uima-website/docs/eclipse-update-site.html uima/site/trunk/uima-website/docs/external-resources.html uima/site/trunk/uima-website/docs/faq.html uima/site/trunk/uima-website/docs/get-involved.html uima/site/trunk/uima-website/docs/gldv07.html uima/site/trunk/uima-website/docs/gscl09.html uima/site/trunk/uima-website/docs/gscl13.html uima/site/trunk/uima-website/docs/iks09.html uima/site/trunk/uima-website/docs/index.html uima/site/trunk/uima-website/docs/license.html uima/site/trunk/uima-website/docs/lrec08.html uima/site/trunk/uima-website/docs/lsm09.html uima/site/trunk/uima-website/docs/mail-lists.html uima/site/trunk/uima-website/docs/management.html uima/site/trunk/uima-website/docs/maven-design.html uima/site/trunk/uima-website/docs/news.html uima/site/trunk/uima-website/docs/one-time-release-setup.html uima/site/trunk/uima-website/docs/one-time-setup.html uima/site/trunk/uima-website/docs/privacy-policy.html uima/site/trunk/uima-website/docs/project-guidelines.html uima/site/trunk/uima-website/docs/publications.html uima/site/trunk/uima-website/docs/release-before-2.3.1.html uima/site/trunk/uima-website/docs/release.html uima/site/trunk/uima-website/docs/roles.html uima/site/trunk/uima-website/docs/ruta.html uima/site/trunk/uima-website/docs/sandbox.html uima/site/trunk/uima-website/docs/saving-svn-resources.html uima/site/trunk/uima-website/docs/security_report.html uima/site/trunk/uima-website/docs/svn.html uima/site/trunk/uima-website/docs/team-list.html uima/site/trunk/uima-website/docs/testing-builds.html uima/site/trunk/uima-website/docs/toolsServers.html uima/site/trunk/uima-website/docs/uima-specification.html uima/site/trunk/uima-website/docs/uimafit.html uima/site/trunk/uima-website/docs/updating-website.html uima/site/trunk/uima-website/xdocs/stylesheets/project.xml Modified: uima/site/trunk/uima-website/docs/annotators.html URL: http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/annotators.html?rev=1858317=1858316=1858317=diff == --- uima/site/trunk/uima-website/docs/annotators.html (original) +++ uima/site/trunk/uima-website/docs/annotators.html Sun Apr 28 11:18:37 2019 @@ -172,6 +172,8 @@ Doing a UIMA release +https://www.apache.org/security/committers.html; target="_blank">Doing a CVE (Apache) + Eclipse Update Sites Code Conventions Modified: uima/site/trunk/uima-website/docs/apache-board-status.html URL: http://svn.apache.org/viewvc/uima/site/trunk/uima-website/docs/apache-board-status.html?rev=18583
svn commit: r1856760 - /uima/uima-ducc/trunk/uima-ducc-common/src/main/versionJava/org/apache/uima/ducc/common/utils/Version.java
Author: degenaro
Date: Mon Apr 1 17:48:05 2019
New Revision: 1856760
URL: http://svn.apache.org/viewvc?rev=1856760=rev
Log:
UIMA-6019 DUCC 3.0.0 system not backwardly compatible with older CLI
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/versionJava/org/apache/uima/ducc/common/utils/Version.java
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/versionJava/org/apache/uima/ducc/common/utils/Version.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/versionJava/org/apache/uima/ducc/common/utils/Version.java?rev=1856760=1856759=1856760=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/versionJava/org/apache/uima/ducc/common/utils/Version.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/versionJava/org/apache/uima/ducc/common/utils/Version.java
Mon Apr 1 17:48:05 2019
@@ -42,7 +42,7 @@ public class Version
* This version number is included on each CLI request and checked by DUCC.
* Change the value when requests are changed in an incompatible way.
*/
-public final static int cliVersion = 1;
+public final static int cliVersion = 3;
public static void main(String[] args)
{
svn commit: r1855337 - /uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
Author: degenaro
Date: Tue Mar 12 15:53:23 2019
New Revision: 1855337
URL: http://svn.apache.org/viewvc?rev=1855337=rev
Log:
UIMA-5999 DUCC Apache demo deployed in "sim" mode has jobs with dependencies
failing
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
Modified:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py?rev=1855337=1855336=1855337=diff
==
---
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
(original)
+++
uima/uima-ducc/trunk/uima-ducc-examples/src/main/uima-ducc-vm/driver/helper.py
Tue Mar 12 15:53:23 2019
@@ -18,11 +18,9 @@
# under the License.
# ---
-
-
-
import os
import random
+import socket
import datetime
class Helper():
@@ -84,9 +82,12 @@ class Helper():
#
+hostname = socket.gethostname().split('.')[0]
+port = '61617'
+
dictService = {
-1:'UIMA-AS:FixedSleepAE_1:tcp://localhost:61617',
-2:'UIMA-AS:FixedSleepAE_4:tcp://localhost:61617',
+1:'UIMA-AS:FixedSleepAE_1:tcp://'+hostname+':'+port,
+2:'UIMA-AS:FixedSleepAE_4:tcp://'+hostname+':'+port,
3:'CUSTOM:localhost:7175',
}
svn commit: r1855336 - in /uima/uima-ducc/trunk: src/main/resources/default.ducc.properties uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/authentication/GSAAuthenticator.java
Author: degenaro
Date: Tue Mar 12 15:25:42 2019
New Revision: 1855336
URL: http://svn.apache.org/viewvc?rev=1855336=rev
Log:
UIMA-5998 DUCC Web Server (WS) authentication plugin for GSA
Added:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/authentication/GSAAuthenticator.java
(with props)
Modified:
uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
Modified: uima/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1855336=1855335=1855336=diff
==
--- uima/uima-ducc/trunk/src/main/resources/default.ducc.properties (original)
+++ uima/uima-ducc/trunk/src/main/resources/default.ducc.properties Tue Mar 12
15:25:42 2019
@@ -151,6 +151,7 @@ ducc.broker.server.url.decoration = tran
# If unconfigured, the Web Server enforces no authentication.
#ducc.authentication.implementer=org.apache.uima.ducc.ws.authentication.LinuxAuthenticationManager
#ducc.authentication.implementer=org.apache.uima.ducc.ws.authentication.SecureFileAuthenticator
+#ducc.authentication.implementer=org.apache.uima.ducc.ws.authentication.GSAAuthenticator
# Specify users allowed to log in to the web server.
#ducc.authentication.users.include = user1 user2
Added:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/authentication/GSAAuthenticator.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/authentication/GSAAuthenticator.java?rev=1855336=auto
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/authentication/GSAAuthenticator.java
(added)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/authentication/GSAAuthenticator.java
Tue Mar 12 15:25:42 2019
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.ws.authentication;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+
+import org.apache.uima.ducc.common.authentication.AuthenticationResult;
+import org.apache.uima.ducc.common.authentication.IAuthenticationResult;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+
+public class GSAAuthenticator extends AbstractAuthenticator {
+
+ private static DuccLogger duccLogger =
DuccLogger.getLogger(GSAAuthenticator.class);
+ private static DuccId jobid = null;
+
+ private String version = "GSA 1.0";
+
+ @Override
+ public String getVersion() {
+ return version;
+ }
+
+ @Override
+ public String getNotes(String userid) {
+ String retVal = "Specify your GSA login password.";
+ return retVal;
+ }
+
+ @Override
+ public boolean isPasswordChecked() {
+ return true;
+ }
+
+ private String get_ducc_ling() throws Exception {
+ DuccPropertiesResolver dpr =
DuccPropertiesResolver.getInstance();
+ String key = "ducc.agent.launcher.ducc_spawn_path";
+ String value = dpr.getFileProperty(key);
+ if(value == null) {
+ throw new Exception("missing ducc_ling in
ducc.properties?");
+ }
+ return value;
+ }
+
+ private String gsa_login = "/usr/bin/gsa_login";
+ private String gsa_arg1 = "-p";
+ private String gsa_response_prefix = "Successfully authenticated";
+
+ private int login(String user, String pw) throws Exception {
+ String location = "login";
+ int rc_success = 0;
+ int rc_fail = 1;
+ int rc = rc_fail;
+ String duc
svn commit: r1853804 - in /uima/uima-ducc/trunk/src/main/admin: ducc_util.py start_ducc
Author: degenaro
Date: Mon Feb 18 15:07:37 2019
New Revision: 1853804
URL: http://svn.apache.org/viewvc?rev=1853804=rev
Log:
UIMA-5742 Reliable DUCC
> prevent start of DB when automanage == False
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/uima-ducc/trunk/src/main/admin/start_ducc
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1853804=1853803=1853804=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Mon Feb 18 15:07:37 2019
@@ -352,6 +352,10 @@ class DuccUtil(DuccBase):
def db_start(self):
+if(not self.automanage_database):
+print ' (Bypass database start - not automanaged)'
+return False
+
# bypass all of this for the initial delivery
if ( self.db_bypass == True) :
print ' (Bypass database start)'
@@ -408,6 +412,9 @@ class DuccUtil(DuccBase):
def db_stop(self):
try:
+if(not self.automanage_database):
+print ' (Bypass database start - not automanaged)'
+return False
if ( self.db_bypass == True) :
print ' (Bypass database stop)'
return True
Modified: uima/uima-ducc/trunk/src/main/admin/start_ducc
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/start_ducc?rev=1853804=1853803=1853804=diff
==
--- uima/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/uima-ducc/trunk/src/main/admin/start_ducc Mon Feb 18 15:07:37 2019
@@ -79,6 +79,13 @@ class StartDucc(DuccUtil):
if (com in self.local_components):
node = self.localhost
+if(self.automanage_database):
+pass
+else:
+if (com in ['db', 'database']):
+msgs.append(('Unmanaged component', component))
+return msgs
+
if ((com in self.default_components) or ( com == 'agent')) :
msgs.append((node, 'Starting', com))
else:
@@ -278,9 +285,19 @@ class StartDucc(DuccUtil):
components.append('rm')
components.append('sm')
components.append('ws')
-components.append('db')
components.append('broker')
+if(self.automanage_database):
+components.append('db')
else:
+if(self.automanage_database):
+pass
+else:
+if('db' in a):
+ print "Database not automanaged."
+ return
+if('database' in a):
+ print "Database not automanaged."
+ return
components.append(a)
elif o in ( '-n', '--nodelist' ):
nodefiles.append(a)
@@ -343,18 +360,19 @@ class StartDucc(DuccUtil):
print "Limits too low to run DUCC"
sys.exit(1)
-if ( self.automanage_database and ('db' in components) ):
-try:
-if ( not self.db_start() ):
-print "Failed to start or connect to the database."
+if ( self.automanage_database ):
+if ( 'db' in components ):
+try:
+if ( not self.db_start() ):
+print "Failed to start or connect to the database."
+sys.exit(1)
+node = self.get_db_host()
+com = 'database'
+self.db_acct_start(node,com)
+except Exception (e):
+# print e
+print sys.exc_info()[0], "Can't start the database."
sys.exit(1)
-node = self.get_db_host()
-com = 'database'
-self.db_acct_start(node,com)
-except Exception (e):
-# print e
-print sys.exc_info()[0], "Can't start the database."
-sys.exit(1)
if ( len(components) == 1 ):
if ( self.automanage_database and ('db' in components) ):
svn commit: r1853796 - in /uima/uima-ducc/trunk: src/main/admin/db_util.py src/main/admin/ducc_post_install uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
Author: degenaro
Date: Mon Feb 18 13:12:19 2019
New Revision: 1853796
URL: http://svn.apache.org/viewvc?rev=1853796=rev
Log:
UIMA-5742 Reliable DUCC
> ducc_post_install should allow for specification of DB replication level
Modified:
uima/uima-ducc/trunk/src/main/admin/db_util.py
uima/uima-ducc/trunk/src/main/admin/ducc_post_install
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
Modified: uima/uima-ducc/trunk/src/main/admin/db_util.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_util.py?rev=1853796=1853795=1853796=diff
==
--- uima/uima-ducc/trunk/src/main/admin/db_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/db_util.py Mon Feb 18 13:12:19 2019
@@ -141,7 +141,7 @@ def update_cassandra_config(DUCC_HOME, D
os.system(ch_head)
-def configure_database(DUCC_HOME, DUCC_HEAD, java, db_autostart=True,
db_host=None, db_user=None, db_pw=None ):
+def configure_database(DUCC_HOME, DUCC_HEAD, java, db_autostart=True,
db_host=None, db_user=None, db_pw=None, db_replication=None):
# for cassandra:
# in ducc_runtime/cassandra-server/conf we need to update cassandra.yaml
to establish
# the data directories and db connection addresses
@@ -193,6 +193,8 @@ def configure_database(DUCC_HOME, DUCC_H
ret = True
CMD = [java, '-DDUCC_HOME=' + DUCC_HOME,
'org.apache.uima.ducc.database.DbCreate', db_host, db_user, db_pw]
+if(db_replication != None):
+CMD.append(db_replication)
CMD = ' '.join(CMD)
if ( execute(CMD) == 0 ):
print 'Database is initialized.'
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_post_install
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1853796=1853795=1853796=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_post_install Mon Feb 18 13:12:19
2019
@@ -66,6 +66,9 @@ class PostInstall():
print " [-a, --db-automanage] "
print "Specify False if DUCC should not start/stop database."
print ""
+print " [-r, --db-replication] "
+print "Specify database replication level (default = 1)."
+print ""
print " [-m, --db-home] "
print "The location of the database home directory, only
specify when db-automanage is False and the default database directory should
not be used."
print ""
@@ -162,7 +165,7 @@ class PostInstall():
#self.update_property('ducc.database.host', '--disabled--', '#
Database support is disabled')
#return True;
-if ( dbu.configure_database(self.DUCC_HOME, self.ducc_head,
self.path_to_java, self.database_automanage, self.database_host_list,
self.database_user, db_pw) ):
+if ( dbu.configure_database(self.DUCC_HOME, self.ducc_head,
self.path_to_java, self.database_automanage, self.database_host_list,
self.database_user, db_pw, self.database_replication) ):
print 'Configuring DUCC to use the database.'
self.update_property('ducc.service.persistence.impl',
'org.apache.uima.ducc.database.StateServicesDb', '# Service manager
persistence')
@@ -461,6 +464,7 @@ class PostInstall():
self.ducc_head = None
self.keystore_pw = None
self.database_automanage = None
+self.database_replication = None
self.database_home = None
self.database_host_list = None
self.database_user = None
@@ -469,7 +473,7 @@ class PostInstall():
self.path_to_java = None
try:
-opts, args = getopt.getopt(argv, 'a:m:o:u:d:b:j:k:n:h?',
['db-automanage=', 'db-home=', 'db-host-list=', 'db-user=', 'db-password=',
'br-password=', 'jvm=', 'keystore=', 'head-node=', 'help'])
+opts, args = getopt.getopt(argv, 'a:r:m:o:u:d:b:j:k:n:h?',
['db-automanage=', 'db-replication=', 'db-home=', 'db-host-list=', 'db-user=',
'db-password=', 'br-password=', 'jvm=', 'keystore=', 'head-node=', 'help'])
except:
self.usage("Invalid arguments " + ' '.join(argv))
@@ -481,6 +485,8 @@ class PostInstall():
self.database_automanage = True
else:
self.database_automanage = False
+if o in ('-r', '--db-replication'):
+self.database_replication = a
if o in ('-m', '--db-home'):
self.database_home = a
if o in ('-o', '--db-host-list'):
Modified:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbCreate.java
URL:
http://svn.apache.org/v
svn commit: r1853794 - in /uima/uima-ducc/trunk/src/main/admin: db_autostart_delete.py db_autostart_insert.py
Author: degenaro Date: Mon Feb 18 13:02:58 2019 New Revision: 1853794 URL: http://svn.apache.org/viewvc?rev=1853794=rev Log: UIMA-5742 Reliable DUCC > admin tools db_autostart_insert and _delete should accept 'db' as component > type Modified: uima/uima-ducc/trunk/src/main/admin/db_autostart_delete.py uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py Modified: uima/uima-ducc/trunk/src/main/admin/db_autostart_delete.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_autostart_delete.py?rev=1853794=1853793=1853794=diff == --- uima/uima-ducc/trunk/src/main/admin/db_autostart_delete.py (original) +++ uima/uima-ducc/trunk/src/main/admin/db_autostart_delete.py Mon Feb 18 13:02:58 2019 @@ -42,7 +42,7 @@ from ducc_util import DuccUtil class AutostartDelete(DuccUtil): - valid_names = [ 'ag', 'br', 'or', 'pm', 'rm', 'sm', 'ws' ] + valid_names = [ 'ag', 'br', 'or', 'pm', 'rm', 'sm', 'ws', 'db' ] jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI' description = 'Delete an entry from the autostart database table.' Modified: uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py?rev=1853794=1853793=1853794=diff == --- uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py (original) +++ uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py Mon Feb 18 13:02:58 2019 @@ -42,7 +42,7 @@ from ducc_util import DuccUtil class AutostartInsert(DuccUtil): - valid_names = [ 'ag', 'br', 'or', 'pm', 'rm', 'sm', 'ws' ] + valid_names = [ 'ag', 'br', 'or', 'pm', 'rm', 'sm', 'ws', 'db' ] jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI' description = 'Insert an entry into the autostart database table, if one does not already exist.'
svn commit: r1853718 - /uima/uima-ducc/trunk/src/main/admin/db_create
Author: degenaro
Date: Sat Feb 16 14:00:22 2019
New Revision: 1853718
URL: http://svn.apache.org/viewvc?rev=1853718=rev
Log:
UIMA-5989 DUCC admin optional tools for starting/stopping unmanaged Cassandra DB
Modified:
uima/uima-ducc/trunk/src/main/admin/db_create
Modified: uima/uima-ducc/trunk/src/main/admin/db_create
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_create?rev=1853718=1853717=1853718=diff
==
--- uima/uima-ducc/trunk/src/main/admin/db_create (original)
+++ uima/uima-ducc/trunk/src/main/admin/db_create Sat Feb 16 14:00:22 2019
@@ -72,7 +72,7 @@ class DbCreate(DuccUtil):
def main(self, argv):
- self.database_automanage = True
+self.database_automanage = True
self.database_host_list = None
self.database_user = None
self.database_pw = None
@@ -124,7 +124,7 @@ class DbCreate(DuccUtil):
if( self.database_host_list == None ) :
self.database_host_list = self.ducc_properties.get("ducc.head")
-if ( self.database_user == None ):
+ if ( self.database_user == None ):
self.database_user = 'ducc'
if ( self.database_pw == None ):
svn commit: r1853663 - in /uima/uima-ducc/trunk/src/main/admin: db_start.py db_stop.py ducc_util.py
Author: degenaro
Date: Fri Feb 15 20:14:39 2019
New Revision: 1853663
URL: http://svn.apache.org/viewvc?rev=1853663=rev
Log:
UIMA-5989 DUCC admin optional tools for starting/stopping unmanaged Cassandra DB
Added:
uima/uima-ducc/trunk/src/main/admin/db_start.py (with props)
uima/uima-ducc/trunk/src/main/admin/db_stop.py (with props)
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Added: uima/uima-ducc/trunk/src/main/admin/db_start.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/db_start.py?rev=1853663=auto
==
--- uima/uima-ducc/trunk/src/main/admin/db_start.py (added)
+++ uima/uima-ducc/trunk/src/main/admin/db_start.py Fri Feb 15 20:14:39 2019
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# ---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ---
+
+import sys
+
+version_min = [2, 7]
+version_info = sys.version_info
+version_error = False
+if(version_info[0] < version_min[0]):
+ version_error = True
+elif(version_info[0] == version_min[0]):
+ if(version_info[1] < version_min[1]):
+ version_error = True
+if(version_error):
+ print('Python minimum requirement is version ' + str(version_min[0]) +
'.' + str(version_min[1]))
+ sys.exit(1)
+
+import argparse
+from argparse import RawDescriptionHelpFormatter
+import os
+import subprocess
+import traceback
+
+from ducc_util import DuccUtil
+
+# command to start DB on local head node, regardless of automanage
+
+class DbStart(DuccUtil):
+
+ def __init__(self):
+ DuccUtil.__init__(self, True)
+
+ # extra help!
+ def get_epilog(self):
+ epilog = ''
+ epilog = epilog+'Purpose: start database on local node,
regardless of automanage.'
+ return epilog
+
+ help_debug = 'Display debugging messages.'
+
+ def debug(self,text):
+ if(self.args.debug):
+ if(text != None):
+ print text
+
+ def main(self, argv):
+ rc = 0
+ try:
+ self.parser =
argparse.ArgumentParser(formatter_class=RawDescriptionHelpFormatter,epilog=self.get_epilog())
+ self.parser.add_argument('--debug', '-d',
action='store_true', help=self.help_debug)
+ self.args = self.parser.parse_args()
+ if(self.db_process_alive()):
+ self.debug('already running')
+ else:
+ here = os.getcwd()
+ self.verify_limits()
+ xmx =
self.ducc_properties.get('ducc.database.mem.heap')
+ new =
self.ducc_properties.get('ducc.database.mem.new')
+ if ( not ( xmx == None and new == None ) ): #
if either is set
+ if ( xmx == None or new == None ) :
# then both must be set
+ self.debug('Database properties
ducc.database.mem.heap and ducc.database.mem.new must both be set.')
+ rc = 1
+ else:
+ os.environ['MAX_HEAP_SIZE'] =
xmx
+ os.environ['HEAP_NEWSIZE'] = new
+ if(rc == 0):
+ jmxport =
self.ducc_properties.get('ducc.database.jmx.port')
+ if ( jmxport != None ):
+ os.environ['JMX_PORT'] = jmxport
+ jmxhost =
self.ducc_properties.get('ducc.database.jmx.host')
+ if ( jmxhost != None and jmxhost !=
'localhost' ):
+
svn commit: r1853651 - /uima/uima-ducc/trunk/src/main/admin/stop_ducc
Author: degenaro Date: Fri Feb 15 15:17:09 2019 New Revision: 1853651 URL: http://svn.apache.org/viewvc?rev=1853651=rev Log: UIMA-5742 Reliable DUCC > stop_ducc -a should be accepted syntax. Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1853651=1853650=1853651=diff == --- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Fri Feb 15 15:17:09 2019 @@ -248,7 +248,7 @@ class StopDucc(DuccUtil): def get_args(self): self.parser = argparse.ArgumentParser(formatter_class=RawDescriptionHelpFormatter,epilog=self.get_epilog()) group1 = self.parser.add_mutually_exclusive_group(required=True) -group1.add_argument(self.option_all, action='store_true', help=self.help_all) +group1.add_argument(self.option_all, '-a', action='store_true', help=self.help_all) group1.add_argument(self.option_head, action='store_true', help=self.help_head) group1.add_argument(self.option_agents, action='store_true', help=self.help_agents) group1.add_argument(self.option_nodelist, '-n', action='append', help=self.help_nodelist)
svn commit: r1853437 - /uima/uima-ducc/trunk/src/main/admin/ducc_update
Author: degenaro
Date: Tue Feb 12 14:00:27 2019
New Revision: 1853437
URL: http://svn.apache.org/viewvc?rev=1853437=rev
Log:
UIMA-5985 ducc_update needs to employ check_ducc like functionality to prevent
updating when DUCC is already running
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_update
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_update
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_update?rev=1853437=1853436=1853437=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_update (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_update Tue Feb 12 14:00:27 2019
@@ -45,6 +45,7 @@ import datetime
import fnmatch
import re
import shutil
+import subprocess
global preserveFiles
global preserveDirectories
@@ -207,7 +208,26 @@ def update_directory(olddir, newdir, arc
if nleft == 0:
os.rmdir(newdir)
-
+#-
+# Insure ducc not running
+#-
+def check_ducc(runtime):
+print 'checking for ducc running, may take a few minutes...'
+cmd = [ os.path.join(runtime,'admin/check_ducc') ]
+p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
bufsize=1, universal_newlines=True)
+while True:
+output = p.stdout.readline()
+if output == '' and p.poll() is not None:
+break
+if output:
+line = output.strip()
+#print line
+if('Found ducc' in line):
+print "ERROR - DUCC appears to be running ... please run '" +
os.path.join(runtime,'admin/stop_ducc') + " -a'"
+exit(1)
+rc = p.poll()
+return rc
+
#=
# Main program:
#tarfile runtime
@@ -246,6 +266,8 @@ rtime = sys.argv[1]
tarfile = sys.argv[2]
runtime = os.path.realpath(rtime)
+check_ducc(runtime)
+
if not os.path.exists(tarfile):
print "ERROR - Missing tar file", tarfile
exit(1)
svn commit: r1853367 - /uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
Author: degenaro
Date: Mon Feb 11 13:07:05 2019
New Revision: 1853367
URL: http://svn.apache.org/viewvc?rev=1853367=rev
Log:
UIMA-5742 Reliable DUCC
> minor doc fix-ups
> autostart.py don't use ssh for utility local host operations (e.g.
> query/start agent)
> ducc_util.py don't create database directories when automanage == false
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex?rev=1853367=1853366=1853367=diff
==
---
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
(original)
+++
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
Mon Feb 11 13:07:05 2019
@@ -54,7 +54,7 @@
\begin{minipage}{\textwidth}
%
- Sample /etc/sysconfig/keepalived
+ Sample /etc/sysconfig/keepalived\\
\begin{greybatim}
@@ -92,7 +92,7 @@ KEEPALIVED_OPTIONS="-D -P"
\begin{minipage}{\textwidth}
%
- Sample MASTER /etc/keepalived/keepalived.conf
+ Sample MASTER /etc/keepalived/keepalived.conf\\
\begin{greybatim}
@@ -138,7 +138,7 @@ vrrp_instance VI_1 {
\begin{minipage}{\textwidth}
%
- Sample BACKUP /etc/keepalived/keepalived.conf
+ Sample BACKUP /etc/keepalived/keepalived.conf\\
\begin{greybatim}
@@ -195,7 +195,7 @@ vrrp_instance VI_1 {
\begin{minipage}{\textwidth}
%
- Sample /etc/keepalived/keepalived\_evaluator.py
+ Sample /etc/keepalived/keepalived\_evaluator.py\\
\begin{greybatim}
@@ -398,18 +398,14 @@ if __name__ == "__main__":
The {\em master} DUCC Webserver will display all pages normally with
additional
information in the heading upper left:
- \begin{verbatim}
reliable: \textbf{master}
- \end{verbatim}
-
+
Webserver for Backup
The {\em backup} DUCC Webserver will display some pages normally with
additional
information in the heading upper left:
- \begin{verbatim}
\underline{reliable}: \textbf{backup}
- \end{verbatim}
Hovering over \underline{reliable} will yield the following information:
{\em Click to visit master}
svn commit: r1853161 - in /uima/uima-ducc/trunk/src/main/admin: autostart.py db_autostart_insert.py ducc_util.py
Author: degenaro
Date: Thu Feb 7 21:33:32 2019
New Revision: 1853161
URL: http://svn.apache.org/viewvc?rev=1853161=rev
Log:
UIMA-5742 Reliable DUCC
> autostart.py insert host+agent start into DB of never before seen node (one
> not already in DB)
> autostart.py don't use ssh for utility local host operations (e.g.
> query/start agent)
> ducc_util.py don't create database directories when automanage == false
Added:
uima/uima-ducc/trunk/src/main/admin/db_autostart_insert.py (with props)
Modified:
uima/uima-ducc/trunk/src/main/admin/autostart.py
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Modified: uima/uima-ducc/trunk/src/main/admin/autostart.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/autostart.py?rev=1853161=1853160=1853161=diff
==
--- uima/uima-ducc/trunk/src/main/admin/autostart.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/autostart.py Thu Feb 7 21:33:32 2019
@@ -114,6 +114,10 @@ class AutoStart(DuccUtil):
'ws':'ws',
}
+def __init__(self):
+DuccUtil.__init__(self)
+self.ssh_enabled = False
+
# return file name
def _fn(self):
fpath = __file__.split('/')
@@ -144,7 +148,6 @@ class AutoStart(DuccUtil):
LOGLEVEL = os.environ.get('LOGLEVEL','info')
self.logger = Logger(LOGFILE,LOGLEVEL)
-
# check if host names with domain match
def is_host_match_with_domain(self,h1,h2):
retVal = False
@@ -195,9 +198,10 @@ class AutoStart(DuccUtil):
pass
return retVal
-# get daemons started (from DB)
-def get_daemons_started(self):
-daemons = []
+# get daemons started/all in DB for host
+def get_daemons_host(self):
+db = []
+started = []
jclass = 'org.apache.uima.ducc.database.lifetime.DbDaemonLifetimeUI'
option = '--query'
cmd = [self.jvm, '-DDUCC_HOME='+self.DUCC_HOME, jclass, option]
@@ -207,8 +211,9 @@ class AutoStart(DuccUtil):
for line in lines:
host, daemon, state = self.parse_line(line)
if(self.is_host_match(self.LOCAL_HOST, host)):
+db.append(daemon)
if(state == 'Start'):
-daemons.append(daemon)
+started.append(daemon)
text = 'add'+' '+host+' '+daemon
self.logger.debug(self._mn(),text)
else:
@@ -217,16 +222,16 @@ class AutoStart(DuccUtil):
else:
text = 'skip'+' '+host+' '+daemon
self.logger.debug(self._mn(),text)
-text = 'daemons'+' '+str(daemons)
+text = 'started'+' '+str(started)+' '+'db'+' '+str(db)
self.logger.debug(self._mn(),text)
-return daemons
+return db, started
def normalize_component(self,component):
daemon = component[:2]
return daemon
# get daemons running (from system)
-def get_daemons_running(self):
+def get_components_running(self):
daemons = []
result = self.find_ducc_process(self.LOCAL_HOST)
find_status = result[0]
@@ -271,19 +276,50 @@ class AutoStart(DuccUtil):
out, err = p.communicate()
text = str(out)
self.logger.info(self._mn(),text)
-
+
+def insert(self):
+python_script =
os.path.join(self.DUCC_HOME,'admin','db_autostart_insert.py')
+node = self.get_node_name()
+component = 'ag'
+cmd = [ python_script, '--host', node, '--name', component]
+text = str(cmd)
+self.logger.info(self._mn(),text)
+p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
+out, err = p.communicate()
+text = str(out)
+self.logger.info(self._mn(),text)
+
# autostart: start head or agent daemons, as required
def main(self, argv):
NAME = 'autostart'
self.setup_logging(NAME)
self.get_args()
try:
-daemons_started = self.get_daemons_started()
-daemons_running = self.get_daemons_running()
+daemons_db, daemons_started = self.get_daemons_host()
+text = 'daemons_db '+str(len(daemons_db))
+self.logger.debug(self._mn(),text)
+text = 'daemons_started '+str(len(daemons_started))
+self.logger.debug(self._mn(),text)
+# if agent node is not in db, then insert it!
+if(len(daemons_db) == 0):
+if(self.is_head_node()):
+pass
+else:
+self.insert()
+daemons_db, daemons_started = self.get_daemons_host()
+components_running = self.get_components_running()
+text = 'components_running '+str(components_running)
+self.logger.debug(
svn commit: r1852935 - /uima/uima-ducc/trunk/src/main/assembly/bin.xml
Author: degenaro Date: Mon Feb 4 16:00:23 2019 New Revision: 1852935 URL: http://svn.apache.org/viewvc?rev=1852935=rev Log: UIMA-5742 Reliable DUCC > provide keepalived examples of key configuration files and monitoring script Modified: uima/uima-ducc/trunk/src/main/assembly/bin.xml Modified: uima/uima-ducc/trunk/src/main/assembly/bin.xml URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1852935=1852934=1852935=diff == --- uima/uima-ducc/trunk/src/main/assembly/bin.xml (original) +++ uima/uima-ducc/trunk/src/main/assembly/bin.xml Mon Feb 4 16:00:23 2019 @@ -581,6 +581,14 @@ under the License. + + uima-ducc-examples/src/main/reliable + examples/reliable + 755 + 755 + + + uima-ducc-examples/src/main/uima-ducc-vm examples/uima-ducc-vm
svn commit: r1852934 - in /uima/uima-ducc/trunk: uima-ducc-duccdocs/src/site/tex/duccbook/part4/ uima-ducc-examples/ uima-ducc-examples/src/main/reliable/ uima-ducc-examples/src/main/reliable/etc/ uim
Author: degenaro
Date: Mon Feb 4 15:27:45 2019
New Revision: 1852934
URL: http://svn.apache.org/viewvc?rev=1852934=rev
Log: (empty)
Added:
uima/uima-ducc/trunk/uima-ducc-examples/.pydevproject
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/etc/
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/etc/keepalived/
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/etc/keepalived/keepalived.conf
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/etc/keepalived/keepalived_evaluator.py
(with props)
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/etc/sysconfig/
uima/uima-ducc/trunk/uima-ducc-examples/src/main/reliable/etc/sysconfig/keepalived
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
Modified:
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex?rev=1852934=1852933=1852934=diff
==
---
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
(original)
+++
uima/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/reliable.tex
Mon Feb 4 15:27:45 2019
@@ -16,11 +16,19 @@
% specific language governing permissions and limitations
% under the License.
%
+
+\let\oldv\verbatim
+\let\oldendv\endverbatim
+
+\def\greybatim{\par\setbox0\vbox\bgroup\oldv}
+\def\endgreybatim{\oldendv\egroup\fboxsep0pt
\noindent\colorbox[gray]{0.8}{\usebox0}\par}
+
% Create well-known link to this spot for HTML version
\ifpdf
\else
\HCode{}
\fi
+
\chapter{Reliable DUCC}
\label{chap:reliable}
@@ -42,48 +50,304 @@
which can be found at this web address:
\url{https://docs.oracle.com/cd/E37670_01/E41138/html/section_uxg_lzh_nr.html}.
- Sample MASTER /etc/keepalived/keepalived.conf
+%
+\begin{minipage}{\textwidth}
+%
+
+ Sample /etc/sysconfig/keepalived
+
+ \begin{greybatim}
- \begin{verbatim}
-! Configuration File for keepalived
+# Options for keepalived. See `keepalived --help' output and keepalived(8) and
+# keepalived.conf(5) man pages for a list of all options. Here are the most
+# common ones :
+#
+# --vrrp -POnly run with VRRP subsystem.
+# --check -COnly run with Health-checker subsystem.
+# --dont-release-vrrp -VDont remove VRRP VIPs & VROUTEs on daemon stop.
+# --dont-release-ipvs -IDont remove IPVS topology on daemon stop.
+# --dump-conf -dDump the configuration data.
+# --log-detail -DDetailed log messages.
+# --log-facility -S0-7 Set local syslog facility (default=LOG_DAEMON)
+#
- vrrp_instance VI_1 {
- state MASTER
- interface eth0
- virtual_router_id 51
- priority 100
- advert_int 1
- authentication {
- auth_type PASS
- auth_pass
- }
- virtual_ipaddress {
- 192.168.6.253
- }
- }
+KEEPALIVED_OPTIONS="-D -P"
+
+ \end{greybatim}
+
+ \medskip
+ IMPORTANT: Specify {\em -P} to enable keepalived IP takeover.
+
+ \medskip
+ See example here:
+ \begin{verbatim}
+examples/reliable/etc/sysconfig/keepalived
\end{verbatim}
+
+%
+\end{minipage}
+%
+
+%
+\begin{minipage}{\textwidth}
+%
+
+ Sample MASTER /etc/keepalived/keepalived.conf
+
+ \begin{greybatim}
+
+! Configuration File for keepalived
+
+global_defs {
+ script_user ducc
+ enable_script_security
+}
+
+vrrp_script chk_ducc {
+ script "/etc/keepalived/keepalived_evaluator.py"
+ interval 5 # check every 5 seconds
+ fall 2 # require 2 failures for KO
+ rise 1 # require 1 successes for OK
+}
+
+vrrp_instance VI_1 {
+state MASTER
+interface bond0
+virtual_router_id 51
+priority 100
+advert_int 1
+authentication {
+auth_type PASS
+auth_pass
+}
+virtual_ipaddress {
+9.59.193.8
+}
+track_script {
+ chk_ducc
+ }
+}
+
+ \end{greybatim}
+
+%
+\end{minipage}
+%
+
+%
+\begin{minipage}{\textwidth}
+%
Sample BACKUP /etc/keepalived/keepalived.conf
- \begin{verbatim}
-! Configura
svn commit: r1852757 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper: BrokerHelper.java DatabaseHelper.java JmxHelper.java
Author: degenaro
Date: Fri Feb 1 15:39:08 2019
New Revision: 1852757
URL: http://svn.apache.org/viewvc?rev=1852757=rev
Log:
UIMA-5977 DUCC Web Server (WS) JMX thread leak
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/DatabaseHelper.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/JmxHelper.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java?rev=1852757=1852756=1852757=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/helper/BrokerHelper.java
Fri Feb 1 15:39:08 2019
@@ -32,13 +32,9 @@ import java.util.TreeSet;
import javax.management.Attribute;
import javax.management.AttributeList;
import javax.management.InstanceNotFoundException;
-import javax.management.MBeanServerConnection;
import javax.management.ObjectName;
import javax.management.ReflectionException;
import javax.management.openmbean.CompositeData;
-import javax.management.remote.JMXConnector;
-import javax.management.remote.JMXConnectorFactory;
-import javax.management.remote.JMXServiceURL;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
@@ -55,10 +51,6 @@ public class BrokerHelper extends JmxHel
return new BrokerHelper();
}
- private JMXServiceURL jmxServiceUrl = null;;
- private JMXConnector jmxc;
- private MBeanServerConnection mbsc;
-
private OperatingSystemMXBean remoteOperatingSystem;
private ThreadMXBean remoteThread;
@@ -133,7 +125,7 @@ public class BrokerHelper extends JmxHel
String location = "populateRemoteOperatingSystem";
try {
remoteOperatingSystem =
ManagementFactory.newPlatformMXBeanProxy(
- mbsc,
+ getMBSC(),
ManagementFactory.OPERATING_SYSTEM_MXBEAN_NAME,
OperatingSystemMXBean.class);
}
@@ -146,7 +138,7 @@ public class BrokerHelper extends JmxHel
String location = "populateRemoteThread";
try {
remoteThread = ManagementFactory.newPlatformMXBeanProxy(
- mbsc,
+ getMBSC(),
ManagementFactory.THREAD_MXBEAN_NAME,
ThreadMXBean.class);
}
@@ -179,7 +171,7 @@ public class BrokerHelper extends JmxHel
private void populateMemory() {
String location = "populateMemory";
try {
- Object o = mbsc.getAttribute(new
ObjectName("java.lang:type=Memory"), "HeapMemoryUsage");
+ Object o = getMBSC().getAttribute(new
ObjectName("java.lang:type=Memory"), "HeapMemoryUsage");
CompositeData cd = (CompositeData) o;
memoryUsed = (Long) cd.get("used");
memoryMax = (Long) cd.get("max");
@@ -193,7 +185,7 @@ public class BrokerHelper extends JmxHel
String location = "populateRuntime";
try {
Object o;
- o = mbsc.getAttribute(new
ObjectName("java.lang:type=Runtime"), "StartTime");
+ o = getMBSC().getAttribute(new
ObjectName("java.lang:type=Runtime"), "StartTime");
startTime = (Long) o;
}
catch(Exception e) {
@@ -301,7 +293,7 @@ public class BrokerHelper extends JmxHel
if(dName.startsWith("ducc.")) {
logger.trace(location, jobid, dType+": "+dName);
Map attributes = new TreeMap();
- AttributeList
attributeList = mbsc.getAttributes(objectName, topicAttributeNames);
+ AttributeList
attributeList = getMBSC().getAttributes(objectName, topicAttributeNames);
svn commit: r1852642 - /uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbOrchestratorProperties.java
Author: degenaro
Date: Thu Jan 31 20:12:34 2019
New Revision: 1852642
URL: http://svn.apache.org/viewvc?rev=1852642=rev
Log:
UIMA-5742 Reliable DUCC
> if OR properties in DB is empty, then initialize:
- duccwork_seqno
- publication_seqno
Modified:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbOrchestratorProperties.java
Modified:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbOrchestratorProperties.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbOrchestratorProperties.java?rev=1852642=1852641=1852642=diff
==
---
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbOrchestratorProperties.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbOrchestratorProperties.java
Thu Jan 31 20:12:34 2019
@@ -152,6 +152,28 @@ public class DbOrchestratorProperties im
}
/**
+* Add or update an Orchestrator property
+*/
+ private void insert(String name, String value) throws Exception {
+ String location = "insert";
+ try {
+ String table = ORCHESTRATOR_PROPERTIES_TABLE;
+ String c0 = COL_VALUE+"="+"'"+value+"'";
+ String c1 = COL_NAME+"="+"'"+name+"'";
+ String cql = "UPDATE "+table+" SET "+c0+" WHERE "+c1;
+ logger.debug(location, jobid, cql);
+ DbHandle h = dbManager.open();
+ h.execute(cql);
+ }
+ catch(Exception e) {
+ DuccId duccid = null;
+ String text = "name="+name+" "+"value="+value;
+ logger.error(location, duccid, text, e);
+ throw e;
+ }
+ }
+
+ /**
* Retrieve an Orchestrator property
*/
@Override
@@ -185,12 +207,19 @@ public class DbOrchestratorProperties im
}
private void initPublicationSeqNo() {
+ String location = "initPublicationSeqNo";
try {
String value = fetch(keys.publication_seqno.name());
Long.parseLong(value);
}
catch(Exception e) {
- setPublicationSeqNo(0);
+ try {
+ long value = 0;
+ insert(keys.publication_seqno.name(),""+value);
+ }
+ catch(Exception x) {
+ logger.error(location, jobid, x);
+ }
}
}
@@ -249,7 +278,13 @@ public class DbOrchestratorProperties im
lval = Long.parseLong(value);
}
catch(Exception e) {
- setDuccWorkSeqNo(0);
+ try {
+ long value = 0;
+ insert(keys.duccwork_seqno.name(),""+value);
+ }
+ catch(Exception x) {
+ logger.error(location, jobid, x);
+ }
}
logger.debug(location, jobid, lval);
}
svn commit: r1852638 - /uima/uima-ducc/trunk/src/main/scripts/ducc_status
Author: degenaro Date: Thu Jan 31 20:06:57 2019 New Revision: 1852638 URL: http://svn.apache.org/viewvc?rev=1852638=rev Log: UIMA-5742 Reliable DUCC > ducc_status should include JD allocation pending as "up", otherwise > oscillation between head nodes occurs Modified: uima/uima-ducc/trunk/src/main/scripts/ducc_status Modified: uima/uima-ducc/trunk/src/main/scripts/ducc_status URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/scripts/ducc_status?rev=1852638=1852637=1852638=diff == --- uima/uima-ducc/trunk/src/main/scripts/ducc_status (original) +++ uima/uima-ducc/trunk/src/main/scripts/ducc_status Thu Jan 31 20:06:57 2019 @@ -291,6 +291,8 @@ def summarize(): value = options.daemons_head[key] if(value == 'up'): options.head_up = options.head_up + 1 +elif(value == 'up, pending JD allocation'): + options.head_up = options.head_up + 1 else: options.head_down = options.head_down + 1 for key in options.daemons_agent:
svn commit: r1852474 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Author: degenaro
Date: Tue Jan 29 20:15:05 2019
New Revision: 1852474
URL: http://svn.apache.org/viewvc?rev=1852474=rev
Log:
UIMA-5742 Reliable DUCC
> WS header: show floating ip address and localhost name on master hover
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1852474=1852473=1852474=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Tue Jan 29 20:15:05 2019
@@ -64,6 +64,7 @@ import org.apache.uima.ducc.common.utils
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.common.utils.DuccSchedulerClasses;
import org.apache.uima.ducc.common.utils.IDuccLoggerComponents;
+import org.apache.uima.ducc.common.utils.InetHelper;
import org.apache.uima.ducc.common.utils.TimeStamp;
import org.apache.uima.ducc.common.utils.Version;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -3405,7 +3406,9 @@ public class DuccHandler extends DuccAbs
duccLogger.trace(methodName, null, messages.fetch("enter"));
StringBuffer sb = new StringBuffer();
String status = dh.get_ducc_head_mode();
- String hover = DuccPropertiesHelper.getDuccHead();
+ String head = "head="+DuccPropertiesHelper.getDuccHead();
+ String local = "local="+InetHelper.getHostName();;
+ String hover = head+' '+local;
if(dh.is_ducc_head_master()) {
String text = ""+status+"";
sb.append(text);
svn commit: r1852177 - /uima/uima-ducc/trunk/src/main/admin/stop_ducc
Author: degenaro Date: Fri Jan 25 20:36:19 2019 New Revision: 1852177 URL: http://svn.apache.org/viewvc?rev=1852177=rev Log: UIMA-5742 Reliable DUCC > default to --stop Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1852177=1852176=1852177=diff == --- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Fri Jan 25 20:36:19 2019 @@ -253,13 +253,18 @@ class StopDucc(DuccUtil): group1.add_argument(self.option_agents, action='store_true', help=self.help_agents) group1.add_argument(self.option_nodelist, '-n', action='append', help=self.help_nodelist) group1.add_argument(self.option_component, '-c', action='append', help=self.help_component) -group2 = self.parser.add_mutually_exclusive_group(required=True) +group2 = self.parser.add_mutually_exclusive_group() group2.add_argument(self.option_kill, '-k', action='store_true', help=self.help_kill) group2.add_argument(self.option_stop, '-s', action='store', type=int, nargs='?', const=self.default_stop, help=self.help_stop) group2.add_argument(self.option_quiesce, '-q', action='store_true', help=self.help_quiesce) self.parser.add_argument(self.option_maxthreads, '-m', action='store', type=int, default=None, help=self.help_maxthreads) self.parser.add_argument(self.option_debug, '-d', action='store_true', help=self.help_debug) self.args = self.parser.parse_args() +# mutual choice +if(not self.args.kill): +if(not self.args.quiesce_then_stop): +if(self.args.stop == None): +self.args.stop = self.default_stop # special cases if(self.args.kill): if(self.args.maxthreads == None):
svn commit: r1852147 - /uima/uima-ducc/trunk/src/main/admin/ducc_post_install
Author: degenaro
Date: Fri Jan 25 15:55:30 2019
New Revision: 1852147
URL: http://svn.apache.org/viewvc?rev=1852147=rev
Log:
UIMA-5742 Reliable DUCC
> facilitate specification of broker pw for ducc_post_install
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_post_install
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_post_install
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1852147=1852146=1852147=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_post_install (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_post_install Fri Jan 25 15:55:30
2019
@@ -78,6 +78,9 @@ class PostInstall():
print " [-d, --db-password] "
print "This is the password DUCC uses to manage the database."
print ""
+print " [-b, --br-password] "
+print "This is the password DUCC uses to manage the broker."
+print ""
print " [-h, -? --help]"
print "Prints this message."
print ""
@@ -111,6 +114,8 @@ class PostInstall():
def configure_broker(self):
cf =
self.DUCC_HOME+"/resources.private/ducc-broker-credentials.properties"
# create file if it does not exist
+if(self.broker_pw == None):
+self.broker_pw = self.generate_pw()
if ( not os.path.exists(cf) ):
# create file with username & password
print "broker configuration create"
@@ -118,7 +123,7 @@ class PostInstall():
line = 'ducc.broker.admin.username=admin'+'\n'
print line
f.write(line)
-line = 'ducc.broker.admin.password='+self.generate_pw()+'\n'
+line = 'ducc.broker.admin.password='+self.broker_pw+'\n'
print line
f.write(line)
# update existing file
@@ -134,7 +139,7 @@ class PostInstall():
line = 'ducc.broker.admin.username=admin'+'\n'
print line
if 'ducc.broker.admin.password=' in line:
-line =
'ducc.broker.admin.password='+self.generate_pw()+'\n'
+line =
'ducc.broker.admin.password='+self.broker_pw+'\n'
print line
f.write(line)
return
@@ -460,10 +465,11 @@ class PostInstall():
self.database_host_list = None
self.database_user = None
self.database_pw = None
+self.broker_pw = None
self.path_to_java = None
try:
-opts, args = getopt.getopt(argv, 'a:m:o:u:d:j:k:n:h?',
['db-automanage=', 'db-home=', 'db-host-list=', 'db-user=', 'db-password=',
'jvm=', 'keystore=', 'head-node=', 'help'])
+opts, args = getopt.getopt(argv, 'a:m:o:u:d:b:j:k:n:h?',
['db-automanage=', 'db-home=', 'db-host-list=', 'db-user=', 'db-password=',
'br-password=', 'jvm=', 'keystore=', 'head-node=', 'help'])
except:
self.usage("Invalid arguments " + ' '.join(argv))
@@ -483,6 +489,8 @@ class PostInstall():
self.database_user = a
if o in ('-d', '--db-password'):
self.database_pw = a
+if o in ('-b', '--br-password'):
+self.broker_pw = a
if o in ('-k', '--keystore'):
self.keystore_pw = a
if o in ('-j', '--jvm'):
svn commit: r1851857 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Author: degenaro
Date: Tue Jan 22 19:49:50 2019
New Revision: 1851857
URL: http://svn.apache.org/viewvc?rev=1851857=rev
Log:
UIMA-5742 Reliable DUCC
> provide WS servlet to return ducc hostname
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1851857=1851856=1851857=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Tue Jan 22 19:49:50 2019
@@ -140,6 +140,7 @@ public class DuccHandler extends DuccAbs
private String duccVersion
= duccContext+"/version";
private String duccHome
= duccContext+"/home";
+ private String duccHostname
= duccContext+"/hostname";
private String duccLoginLink=
duccContext+"/login-link";
private String duccLogoutLink =
duccContext+"/logout-link";
@@ -304,6 +305,27 @@ public class DuccHandler extends DuccAbs
response.getWriter().println(sb);
duccLogger.trace(methodName, null, messages.fetch("exit"));
}
+
+ private void handleDuccServletHostname(String target,Request
baseRequest,HttpServletRequest request,HttpServletResponse response)
+ throws IOException, ServletException
+ {
+ String methodName = "handleDuccServletHostname";
+ duccLogger.trace(methodName, null, messages.fetch("enter"));
+ StringBuffer sb = new StringBuffer();
+ try {
+ Process p = Runtime.getRuntime().exec("hostname");
+ BufferedReader stdInput = new BufferedReader(new
InputStreamReader(p.getInputStream()));
+ String hostname = "";
+ while ((hostname = stdInput.readLine()) != null) {
+ sb.append(hostname);
+ }
+ }
+ catch(Exception e) {
+ duccLogger.error(methodName, null, e);
+ }
+ response.getWriter().println(sb);
+ duccLogger.trace(methodName, null, messages.fetch("exit"));
+ }
private void handleDuccServletAuthenticationStatus(String
target,Request baseRequest,HttpServletRequest request,HttpServletResponse
response)
throws IOException, ServletException
@@ -4459,6 +4481,10 @@ public class DuccHandler extends DuccAbs
handleDuccServletHome(target, baseRequest,
request, response);
//DuccWebUtil.noCache(response);
}
+ else if(reqURI.startsWith(duccHostname)) {
+ handleDuccServletHostname(target, baseRequest,
request, response);
+ //DuccWebUtil.noCache(response);
+ }
else if(reqURI.startsWith(duccAuthenticationStatus)) {
handleDuccServletAuthenticationStatus(target,
baseRequest, request, response);
DuccWebUtil.noCache(response);
svn propchange: r1851733 - svn:log
Author: degenaro Revision: 1851733 Modified property: svn:log Modified: svn:log at Mon Jan 21 11:36:22 2019 -- --- svn:log (original) +++ svn:log Mon Jan 21 11:36:22 2019 @@ -0,0 +1 @@ +UIMA-5742 Reliable DUCC
svn commit: r1851733 - /uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Author: degenaro
Date: Mon Jan 21 11:29:31 2019
New Revision: 1851733
URL: http://svn.apache.org/viewvc?rev=1851733=rev
Log: (empty)
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_util.py
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_util.py
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1851733=1851732=1851733=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_util.py Mon Jan 21 11:29:31 2019
@@ -1301,7 +1301,18 @@ class DuccUtil(DuccBase):
retVal = True
break
return retVal
-
+
+def is_valid_ip_address(self,address):
+retVal = False
+try:
+list = address.split('.')
+for item in list:
+int(item)
+retVal = True
+except Exception as e:
+pass
+return retVal
+
# master when current node keepalived answers for head node ip
# backup when current node keepalived does not answer for head ip, but is
capable in config
# unspecified otherwise
@@ -1310,7 +1321,10 @@ class DuccUtil(DuccBase):
result = 'unspecified'
try:
ducc_head = self.ducc_properties.get('ducc.head')
-head_ip = self.get_ip_address(ducc_head)
+if(self.is_valid_ip_address(ducc_head)):
+head_ip = ducc_head
+else:
+head_ip = self.get_ip_address(ducc_head)
if(self.is_reliable_eligible(head_ip)):
text = 'cmd: ', '/sbin/ip', 'addr', 'list'
debug(label, text)
svn commit: r1851716 - /uima/uima-ducc/trunk/src/main/admin/ducc_rsync
Author: degenaro
Date: Sun Jan 20 19:28:36 2019
New Revision: 1851716
URL: http://svn.apache.org/viewvc?rev=1851716=rev
Log:
UIMA-5958 DUCC resources distribution tool
Modified:
uima/uima-ducc/trunk/src/main/admin/ducc_rsync
Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1851716=1851715=1851716=diff
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (original)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Sun Jan 20 19:28:36 2019
@@ -116,13 +116,16 @@ class DuccRsync(DuccUtil):
head = self.ducc_properties.get(self.key_ducc_head)
if(self.args.debug):
print 'head:'+head
-self.add(node_list,head)
#reliable
-reliable = self.ducc_properties.get(self.key_ducc_head_reliable_list)
-for node in reliable:
-if(self.args.debug):
-print 'reliable:'+head
-self.add(node_list,node)
+reliable_string =
self.ducc_properties.get(self.key_ducc_head_reliable_list)
+reliable = reliable_string.split()
+if(len(reliable) > 0):
+for node in reliable:
+if(self.args.debug):
+print 'reliable:'+node
+self.add(node_list,node)
+else:
+self.add(node_list,head)
# agents
map = {}
if(self.args.debug):
svn commit: r1851707 - /uima/uima-ducc/trunk/src/main/admin/ducc_rsync
Author: degenaro Date: Sun Jan 20 16:17:47 2019 New Revision: 1851707 URL: http://svn.apache.org/viewvc?rev=1851707=rev Log: UIMA-5958 DUCC resources distribution tool Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync Modified: uima/uima-ducc/trunk/src/main/admin/ducc_rsync URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1851707=1851706=1851707=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Sun Jan 20 16:17:47 2019 @@ -93,10 +93,10 @@ class DuccRsync(DuccUtil): else: list.append(node) -# update the specified node, dir +# update the specified node, subdir def rsync(self,node,user,subdir): dir = os.path.join(DUCC_HOME,subdir) -cmd = self.rsync_cmd+' '+self.rsync_flags+' '+dir+' '+user+'@'+node+':'+dir +cmd = self.rsync_cmd+' '+self.rsync_flags+' '+dir+' '+user+'@'+node+':'+DUCC_HOME if(self.args.debug): print cmd proc = subprocess.Popen(cmd, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -109,7 +109,7 @@ class DuccRsync(DuccUtil): for line in lines: print line -# create list of node to be updated +# create list of nodes to be updated def create_node_list(self): node_list = [] # head
svn commit: r1851378 - in /uima/uima-ducc/trunk/src/main/admin: ducc_rsync stop_ducc
Author: degenaro
Date: Tue Jan 15 15:22:50 2019
New Revision: 1851378
URL: http://svn.apache.org/viewvc?rev=1851378=rev
Log:
UIMA-5958 DUCC resources distribution tool
Added:
uima/uima-ducc/trunk/src/main/admin/ducc_rsync (with props)
Modified:
uima/uima-ducc/trunk/src/main/admin/stop_ducc
Added: uima/uima-ducc/trunk/src/main/admin/ducc_rsync
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_rsync?rev=1851378=auto
==
--- uima/uima-ducc/trunk/src/main/admin/ducc_rsync (added)
+++ uima/uima-ducc/trunk/src/main/admin/ducc_rsync Tue Jan 15 15:22:50 2019
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# ---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ---
+
+import sys
+
+version_min = [2, 7]
+version_info = sys.version_info
+version_error = False
+if(version_info[0] < version_min[0]):
+version_error = True
+elif(version_info[0] == version_min[0]):
+if(version_info[1] < version_min[1]):
+version_error = True
+if(version_error):
+print('Python minimum requirement is version
'+str(version_min[0])+'.'+str(version_min[1]))
+sys.exit(1)
+
+import argparse
+import os
+
+from argparse import RawDescriptionHelpFormatter
+
+from ducc_util import *
+
+class DuccRsync(DuccUtil):
+
+merge = False
+
+key_ducc_head = 'ducc.head'
+key_ducc_head_reliable_list = 'ducc.head.reliable.list'
+
+rsync_cmd = 'rsync'
+rsync_flags = '-avz --delete --ignore-errors'
+
+def __init__(self):
+DuccUtil.__init__(self, self.merge)
+
+def _fn(self):
+fpath = __file__.split('/')
+flen = len(fpath)
+return fpath[flen-1]
+
+# extra help!
+def get_epilog(self):
+epilog = ''
+epilog = epilog+'Purpose: synchronize DUCC directory(s) from present
node to other head & agent nodes.'
+epilog = epilog+'\n\n'
+epilog = epilog+'Example:'
+epilog = epilog+'\n\n'
+epilog = epilog+'> '+self._fn()+' '+'--dirs "resources admin"
--nodelist ducc.nodes'
+return epilog
+
+default_dirs = 'resources'
+default_nodelist = 'ducc.nodes'
+
+help_dirs= 'Blank separated list of directories to synchronize,
default='+default_dirs+'.'
+help_nodelist= 'Blank separated list of file names,
default='+default_nodelist+'.'
+help_debug = 'Display debugging messages.'
+help_quiet = 'Suppress informational messages.'
+
+# parse command line
+def get_args(self):
+self.parser =
argparse.ArgumentParser(formatter_class=RawDescriptionHelpFormatter,epilog=self.get_epilog())
+self.parser.add_argument('--dirs' , '-d', action='store',
default=self.default_dirs, help=self.help_dirs)
+self.parser.add_argument('--nodelist' , '-n', action='store',
default=self.default_nodelist, help=self.help_nodelist)
+self.parser.add_argument('--debug', '-g', action='store_true',
help=self.help_debug)
+self.parser.add_argument('--quiet', '-q', action='store_true',
help=self.help_quiet)
+self.args = self.parser.parse_args()
+
+# conditionally add node to list of nodes to be updated
+def add(self,list,node):
+if(node == self.localhost):
+pass
+elif(node in list):
+pass
+else:
+list.append(node)
+
+# update the specified node, dir
+def rsync(self,node,user,subdir):
+dir = os.path.join(DUCC_HOME,subdir)
+cmd = self.rsync_cmd+' '+self.rsync_flags+' '+dir+'
'+user+'@'+node+':'+dir
+if(self.args.debug):
+print cmd
+proc = subprocess.Popen(cmd, shell=True, bufsize=0,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+lines = []
+for line in proc.stdout:
+lines.append(line.strip())
+proc.wait()
+rc = proc.returncode
+if(self.args.debug):
+for line in lines:
+print line
+
+# create list of node t
svn commit: r1851162 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root: $copyright-narrow.jsp $copyright.jsp
Author: degenaro Date: Sat Jan 12 12:52:02 2019 New Revision: 1851162 URL: http://svn.apache.org/viewvc?rev=1851162=rev Log: UIMA-5956 DUCC Web Server (WS) copyright 2019 Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright-narrow.jsp uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright.jsp Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright-narrow.jsp URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/%24copyright-narrow.jsp?rev=1851162=1851161=1851162=diff == --- uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright-narrow.jsp (original) +++ uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright-narrow.jsp Sat Jan 12 12:52:02 2019 @@ -20,7 +20,7 @@ under the License. Copyrights: - 2012-2018 The Apache Software Foundation + 2012-2019 The Apache Software Foundation 2011-2012 IBM Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright.jsp URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/%24copyright.jsp?rev=1851162=1851161=1851162=diff == --- uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright.jsp (original) +++ uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/$copyright.jsp Sat Jan 12 12:52:02 2019 @@ -16,4 +16,4 @@ KIND, either express or implied. See th specific language governing permissions and limitations under the License. --> -Copyrights: 2012-2018 The Apache Software Foundation and 2011-2012 IBM \ No newline at end of file +Copyrights: 2012-2019 The Apache Software Foundation and 2011-2012 IBM \ No newline at end of file
svn commit: r1851161 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws: registry/ server/ utils/
Author: degenaro
Date: Sat Jan 12 12:41:59 2019
New Revision: 1851161
URL: http://svn.apache.org/viewvc?rev=1851161=rev
Log:
UIMA-5787 DUCC Web Server (WS) should honor service administrator
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/utils/HandlersHelper.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java?rev=1851161=1851160=1851161=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java
Sat Jan 12 12:41:59 2019
@@ -141,6 +141,35 @@ public class ServicesRegistry {
return retVal;
}
+ public ServicesRegistryMapPayload findServiceById(String id) {
+ String location = "findServiceById";
+ ServicesRegistryMapPayload retVal = null;
+ try {
+ logger.debug(location, jobid, "size: "+map.size());
+ logger.debug(location, jobid, "search: "+id);
+ for(Long key : map.keySet()) {
+ ServicesRegistryMapPayload payload =
map.get(key);
+ Properties meta = payload.meta;
+ if(meta != null) {
+
if(meta.containsKey(IServicesRegistry.numeric_id)) {
+ String sid =
meta.getProperty(IServicesRegistry.numeric_id);
+ if(sid.equals(id)) {
+ retVal = payload;
+ break;
+ }
+ }
+ }
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ if(retVal == null) {
+ logger.warn(location, jobid, "not found: "+id);
+ }
+ return retVal;
+ }
+
public ServicesRegistryMapPayload findService(String name) {
String location = "findService";
ServicesRegistryMapPayload retVal = null;
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java?rev=1851161=1851160=1851161=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
Sat Jan 12 12:41:59 2019
@@ -57,6 +57,7 @@ import org.apache.uima.ducc.ws.DuccDataH
import org.apache.uima.ducc.ws.DuccMachinesData;
import org.apache.uima.ducc.ws.registry.IServicesRegistry;
import org.apache.uima.ducc.ws.registry.ServicesRegistry;
+import org.apache.uima.ducc.ws.registry.sort.IServiceAdapter;
import org.apache.uima.ducc.ws.server.DuccCookies.DateStyle;
import org.apache.uima.ducc.ws.server.IWebMonitor.MonitorType;
import org.apache.uima.ducc.ws.utils.FormatHelper;
@@ -742,7 +743,34 @@ public abstract class DuccAbstractHandle
return getDisabledWithHover(request, resourceOwnerUserId);
}
- public String getDisabledWithHover(HttpServletRequest request, String
resourceOwnerUserId) {
+ public String getDisabledWithHover(HttpServletRequest request, String
resourceOwnerUserId, IServiceAdapter service) {
+ String location = "getDisabledWithHover";
+ String disabled = "disabled=\"disabled\"";
+ String hover = "";
+ try {
+ String u
svn commit: r1850364 - in /uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service: CustomPingNotAlive.java CustomPingNotHealthy.java
Author: degenaro
Date: Fri Jan 4 14:11:13 2019
New Revision: 1850364
URL: http://svn.apache.org/viewvc?rev=1850364=rev
Log: (empty)
Added:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotAlive.java
(with props)
uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotHealthy.java
(with props)
Added:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotAlive.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotAlive.java?rev=1850364=auto
==
---
uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotAlive.java
(added)
+++
uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotAlive.java
Fri Jan 4 14:11:13 2019
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.ducc.test.service;
+
+import java.io.DataInputStream;
+import java.io.InputStream;
+import java.net.Socket;
+
+import org.apache.uima.ducc.cli.AServicePing;
+import org.apache.uima.ducc.cli.ServiceStatistics;
+
+/**
+ * This is designed for the simple sleeper "service" that does nothing other
than
+ * wait for requests from the pinger.
+ *
+ * The necessary endpoint is CUSTOM:name:host:port
+ */
+public class CustomPingNotAlive
+extends AServicePing
+{
+String host;
+String port;
+public void init(String arguments, String endpoint)
+ throws Exception
+{
+String[] parts = endpoint.split(":");
+// [1] is the type of service, always ignored
+host = parts[2];
+port = parts[3];
+}
+
+public void stop()
+{
+}
+
+public long readLong(DataInputStream dis)
+ throws Exception
+{
+long stat1 = dis.readLong();
+return Long.reverseBytes(stat1);
+}
+
+public ServiceStatistics getStatistics()
+{
+ServiceStatistics stats = new ServiceStatistics(false, false,"");
+try {
+Socket sock = new Socket(host, Integer.parseInt(port));
+InputStream sock_in = sock.getInputStream();
+
+DataInputStream dis = new DataInputStream(sock_in);
+
+long stat1 = readLong(dis);
+long stat2 = readLong(dis);
+long stat3 = readLong(dis);
+long stat4 = readLong(dis);
+
+stats.setAlive(false);
+stats.setHealthy(true);
+stats.setInfo( "S1[" + stat1 +
+"] S2[" + stat2 +
+"] S3[" + stat3 +
+"] S4[" + stat4 +
+"]"
+);
+sock.close();
+} catch ( Throwable t) {
+ t.printStackTrace();
+stats.setInfo(t.getMessage());
+}
+return stats;
+}
+
+public static void main(String[] args)
+{
+try {
+ CustomPingNotAlive cp = new CustomPingNotAlive();
+ cp.init(null, args[0]);
+ for ( int i = 0; i < 10; i++ ) {
+ ServiceStatistics stats = cp.getStatistics();
+ System.out.println(stats);
+ Thread.sleep(2000);
+ }
+ cp.stop();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+}
+
+}
Propchange:
uima/uima-ducc/trunk/uima-ducc-examples/src/main/java/org/apache/uima/ducc/test/service/CustomPingNotAlive.java
--
svn commit: r1850235 - in /uima/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/ uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ uima-ducc-rm/src/m
Author: degenaro
Date: Thu Jan 3 13:51:01 2019
New Revision: 1850235
URL: http://svn.apache.org/viewvc?rev=1850235=rev
Log:
UIMA-5928 DUCC Agent quiesce should wait (forever) for non-fairshare
displatchables to complete before shutting down
RM should honor NodeMetrics report of NodeStatus "UnAvailable" as indication of
node "quiesced"
WS should display Machines page Quiesce status: T/F
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/INodeState.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/state/monitoring/NodeState.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java?rev=1850235=1850234=1850235=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/rm/IRmPersistence.java
Thu Jan 3 13:51:01 2019
@@ -259,6 +259,11 @@ public interface IRmPersistence
public Type type() { return Type.Boolean; }
public boolean isIndex() { return true; }
},
+Quiesced{
+public String pname() { return "quiesced"; }
+public Type type() { return Type.Boolean; }
+public boolean isIndex() { return true; }
+},
Reservable{
public String pname() { return "reservable"; }
public Type type() { return Type.Boolean; }
Modified:
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java?rev=1850235=1850234=1850235=diff
==
---
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/event/ResourceManagerEventListener.java
Thu Jan 3 13:51:01 2019
@@ -20,7 +20,10 @@ package org.apache.uima.ducc.rm.event;
import org.apache.camel.Body;
import org.apache.uima.ducc.common.ANodeStability;
+import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.node.metrics.NodeMetrics;
+import org.apache.uima.ducc.common.node.metrics.NodeMetrics.NodeStatus;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.rm.ResourceManager;
@@ -90,8 +93,24 @@ public class ResourceManagerEventListene
*/
public void onNodeMetricsEvent(@Body NodeMetricsUpdateDuccEvent duccEvent)
throws Exception
{
+ String location = "onNodeMetricsEvent";
+ DuccId jobid = null;
//rm.nodeArrives(duccEvent.getNode());
-nodeStability.nodeArrives(duccEvent.getNode());
+ Node node = duccEvent.getNode();
+nodeStability.nodeArrives(node);
+if(node != null) {
+ NodeMetrics nodeMetrics = node.getNodeMetrics();
+ if(nodeMetrics != null) {
+ NodeStatus nodeStatus = nodeMetrics.getNodeStatus();
+ String name = null;
+ NodeIdentity nodeIdentity = node.getNodeIdentity();
+ if(nodeIdentity != null) {
+ name = nodeIdentity.getShortName();
+ }
+ logger.debug(location, jobid, name, nodeStatus.name());
+ }
+
+}
}
public void onNodeInventoryUpdateEvent(@Body NodeInventoryUpdateDuccEvent
duccEvent) throws Exception {
Modified:
uima/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL:
http://svn.apache.org/viewvc/uima/uim
svn commit: r1848697 - in /uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common: ADuccWork.java IDuccWork.java
Author: degenaro
Date: Tue Dec 11 16:47:02 2018
New Revision: 1848697
URL: http://svn.apache.org/viewvc?rev=1848697=rev
Log:
UIMA-5928 DUCC Agent quiesce should wait (forever) for non-fairshare
displatchables to complete before shutting down
Provide method interface and implementation to determine preemptability status
of dispatchable entities, where returned value is boolean or Exception.
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java?rev=1848697=1848696=1848697=diff
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
Tue Dec 11 16:47:02 2018
@@ -95,27 +95,11 @@ public abstract class ADuccWork implemen
this.duccSchedulingInfo = schedulingInfo;
}
- // Evaluate preemptability of scheduling class for entity: { 1 == yes,
-1 == no, 0 == dunno }
- public int getPreemptableStatus() {
- int retVal = 0; // dunno
- try {
- if(duccSchedulingInfo != null) {
- String class_name =
duccSchedulingInfo.getSchedulingClass();
- if(class_name != null) {
- DuccSchedulerClasses
duccSchedulerClasses = DuccSchedulerClasses.getInstance();
- boolean preemptable =
duccSchedulerClasses.isPreemptable(class_name);
- if(preemptable) {
- retVal = 0+1; // yes
- }
- else {
- retVal = 0-1; // no
- }
- }
- }
- }
- catch(Exception e) {
- // dunno
- }
+ // Evaluate preemptability of scheduling class for entity:
+ public boolean getPreemptableStatus() throws Exception {
+ String class_name = duccSchedulingInfo.getSchedulingClass();
+ DuccSchedulerClasses duccSchedulerClasses =
DuccSchedulerClasses.getInstance();
+ boolean retVal = duccSchedulerClasses.isPreemptable(class_name);
return retVal;
}
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java?rev=1848697=1848696=1848697=diff
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
Tue Dec 11 16:47:02 2018
@@ -41,8 +41,8 @@ public interface IDuccWork extends Ident
public IDuccSchedulingInfo getSchedulingInfo();
public void setSchedulingInfo(IDuccSchedulingInfo schedulingInfo);
- // Evaluate preemptability of scheduling class for entity: { 1 == yes,
-1 == no, 0 == dunno }
- public int getPreemptableStatus();
+ // Evaluate preemptability of scheduling class for entity
+ public boolean getPreemptableStatus() throws Exception;
public Object getStateObject();
public void setStateObject(Object state);
svn commit: r1848688 - in /uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common: ADuccWork.java IDuccWork.java
Author: degenaro
Date: Tue Dec 11 14:30:10 2018
New Revision: 1848688
URL: http://svn.apache.org/viewvc?rev=1848688=rev
Log:
UIMA-5928 DUCC Agent quiesce should wait (forever) for non-fairshare
displatchables to complete before shutting down
Provide method interface and implementation to determine preemptability status
of dispatchable entities.
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java?rev=1848688=1848687=1848688=diff
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/ADuccWork.java
Tue Dec 11 14:30:10 2018
@@ -18,6 +18,7 @@
*/
package org.apache.uima.ducc.transport.event.common;
+import org.apache.uima.ducc.common.utils.DuccSchedulerClasses;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
import
org.apache.uima.ducc.transport.event.common.IDuccWorkService.ServiceDeploymentType;
@@ -94,6 +95,29 @@ public abstract class ADuccWork implemen
this.duccSchedulingInfo = schedulingInfo;
}
+ // Evaluate preemptability of scheduling class for entity: { 1 == yes,
-1 == no, 0 == dunno }
+ public int getPreemptableStatus() {
+ int retVal = 0; // dunno
+ try {
+ if(duccSchedulingInfo != null) {
+ String class_name =
duccSchedulingInfo.getSchedulingClass();
+ if(class_name != null) {
+ DuccSchedulerClasses
duccSchedulerClasses = DuccSchedulerClasses.getInstance();
+ boolean preemptable =
duccSchedulerClasses.isPreemptable(class_name);
+ if(preemptable) {
+ retVal = 0+1; // yes
+ }
+ else {
+ retVal = 0-1; // no
+ }
+ }
+ }
+ }
+ catch(Exception e) {
+ // dunno
+ }
+ return retVal;
+ }
public Object getStateObject() {
return stateObject;
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java?rev=1848688=1848687=1848688=diff
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccWork.java
Tue Dec 11 14:30:10 2018
@@ -41,6 +41,9 @@ public interface IDuccWork extends Ident
public IDuccSchedulingInfo getSchedulingInfo();
public void setSchedulingInfo(IDuccSchedulingInfo schedulingInfo);
+ // Evaluate preemptability of scheduling class for entity: { 1 == yes,
-1 == no, 0 == dunno }
+ public int getPreemptableStatus();
+
public Object getStateObject();
public void setStateObject(Object state);
svn commit: r1847537 - in /uima/uima-ducc/trunk/src/main: assembly/bin.xml config/cassandra.yaml.local config/cassandra.yaml.remote
Author: degenaro
Date: Tue Nov 27 13:40:14 2018
New Revision: 1847537
URL: http://svn.apache.org/viewvc?rev=1847537=rev
Log:
UIMA-5742 Reliable DUCC
> Include (for reference) Cassandra yaml templates for local and remote DB
Added:
uima/uima-ducc/trunk/src/main/config/cassandra.yaml.local (with props)
uima/uima-ducc/trunk/src/main/config/cassandra.yaml.remote (with props)
Modified:
uima/uima-ducc/trunk/src/main/assembly/bin.xml
Modified: uima/uima-ducc/trunk/src/main/assembly/bin.xml
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/assembly/bin.xml?rev=1847537=1847536=1847537=diff
==
--- uima/uima-ducc/trunk/src/main/assembly/bin.xml (original)
+++ uima/uima-ducc/trunk/src/main/assembly/bin.xml Tue Nov 27 13:40:14 2018
@@ -348,6 +348,8 @@ under the License.
cassandra-env.sh
cassandra.yaml
+cassandra.yaml.local
+cassandra.yaml.remote
755
755
Added: uima/uima-ducc/trunk/src/main/config/cassandra.yaml.local
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/config/cassandra.yaml.local?rev=1847537=auto
==
--- uima/uima-ducc/trunk/src/main/config/cassandra.yaml.local (added)
+++ uima/uima-ducc/trunk/src/main/config/cassandra.yaml.local Tue Nov 27
13:40:14 2018
@@ -0,0 +1,813 @@
+# Cassandra storage config YAML
+
+#
+# This is prepared for configuration for DUCC via ducc_post_install
+#
+
+# NOTE:
+# See http://wiki.apache.org/cassandra/StorageConfiguration for
+# full explanations of configuration directives
+# /NOTE
+
+# The name of the cluster. This is mainly used to prevent machines in
+# one logical cluster from joining another.
+cluster_name: 'DUCC'
+
+# This defines the number of tokens randomly assigned to this node on the ring
+# The more tokens, relative to other nodes, the larger the proportion of data
+# that this node will store. You probably want all nodes to have the same
number
+# of tokens assuming they have equal hardware capability.
+#
+# If you leave this unspecified, Cassandra will use the default of 1 token for
legacy compatibility,
+# and will use the initial_token as described below.
+#
+# Specifying initial_token will override this setting on the node's initial
start,
+# on subsequent starts, this setting will apply even if initial token is set.
+#
+# If you already have a cluster with 1 token per node, and wish to migrate to
+# multiple tokens per node, see http://wiki.apache.org/cassandra/Operations
+num_tokens: 256
+
+# initial_token allows you to specify tokens manually. While you can use # it
with
+# vnodes (num_tokens > 1, above) -- in which case you should provide a
+# comma-separated list -- it's primarily used when adding nodes # to legacy
clusters
+# that do not have vnodes enabled.
+# initial_token:
+
+# See http://wiki.apache.org/cassandra/HintedHandoff
+# May either be "true" or "false" to enable globally, or contain a list
+# of data centers to enable per-datacenter.
+# hinted_handoff_enabled: DC1,DC2
+hinted_handoff_enabled: true
+# this defines the maximum amount of time a dead host will have hints
+# generated. After it has been dead this long, new hints for it will not be
+# created until it has been seen alive and gone down again.
+max_hint_window_in_ms: 1080 # 3 hours
+# Maximum throttle in KBs per second, per delivery thread. This will be
+# reduced proportionally to the number of nodes in the cluster. (If there
+# are two nodes in the cluster, each delivery thread will use the maximum
+# rate; if there are three, each will throttle to half of the maximum,
+# since we expect two nodes to be delivering hints simultaneously.)
+hinted_handoff_throttle_in_kb: 1024
+# Number of threads with which to deliver hints;
+# Consider increasing this number when you have multi-dc deployments, since
+# cross-dc handoff tends to be slower
+max_hints_delivery_threads: 2
+
+# Maximum throttle in KBs per second, total. This will be
+# reduced proportionally to the number of nodes in the cluster.
+batchlog_replay_throttle_in_kb: 1024
+
+# Authentication backend, implementing IAuthenticator; used to identify users
+# Out of the box, Cassandra provides
org.apache.cassandra.auth.{AllowAllAuthenticator,
+# PasswordAuthenticator}.
+#
+# - AllowAllAuthenticator performs no checks - set it to disable
authentication.
+# - PasswordAuthenticator relies on username/password pairs to authenticate
+# users. It keeps usernames and hashed passwords in system_auth.credentials
table.
+# Please increase system_auth keyspace replication factor if you use this
authenticator.
+authenticator: PasswordAuthenticator
+
+# Authorization backend, implementing IAuthorizer; used to limit
access/provide permissions
+# Out of the box, Cassandra provides
org.apache.cassandr
svn commit: r1847535 - in /uima/uima-ducc/trunk: src/main/admin/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/comp
Author: degenaro
Date: Tue Nov 27 13:31:57 2018
New Revision: 1847535
URL: http://svn.apache.org/viewvc?rev=1847535=rev
Log:
UIMA-5875 DUCC admin command stop_ducc should accept --all --stop, which
comprises --agents followed by --head
Fix targeted stop.
> broadcast of stop_ducc --stop honored only by targeted daemons
Modified:
uima/uima-ducc/trunk/src/main/admin/stop_ducc
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/TargetableDuccAdminEvent.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/component/AbstractDuccComponent.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccAdmin.java
Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1847535=1847534=1847535=diff
==
--- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original)
+++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Tue Nov 27 13:31:57 2018
@@ -286,7 +286,7 @@ class StopDucc(DuccUtil):
def get_db_list(self):
if(self.db_list == None):
self.db_list = self.db_acct_query()
-text = 'list='+str(list)
+text = 'list='+str(self.db_list)
debug(self._mn(),text)
return self.db_list
@@ -525,6 +525,7 @@ class StopDucc(DuccUtil):
admin = ''
stop_db = False
stop_broker = False
+# pass 1 remove db,br
for item in list:
node = item[0]
com = item[1]
@@ -540,15 +541,14 @@ class StopDucc(DuccUtil):
component = self.longname[com]
admin = admin+component+'@'+node+' '
# issue command
-admin = admin.strip()
-if(len(admin) > 0):
-admin = str(self.args.stop)+' '+admin
+admin = admin.split()
+for item in admin:
if(qflag):
-print 'quiesce: '+admin
-self.ducc_admin('--quiesce',admin)
+print 'quiesce: '+item
+self.ducc_admin('--quiesce',item)
else:
-print 'stop: '+admin
-self.ducc_admin('--stop',admin)
+print 'stop: '+item
+self.ducc_admin('--stop',item)
# stop broker
if(stop_broker):
self.stop_broker()
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/TargetableDuccAdminEvent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/TargetableDuccAdminEvent.java?rev=1847535=1847534=1847535=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/TargetableDuccAdminEvent.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/TargetableDuccAdminEvent.java
Tue Nov 27 13:31:57 2018
@@ -36,5 +36,13 @@ public abstract class TargetableDuccAdmi
public String getTargets() {
return this.targetList;
}
+
+ public String[] getTargetList() {
+ String[] retVal = null;
+ if(targetList != null) {
+ retVal = targetList.split(",");
+ }
+ return retVal;
+ }
}
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/component/AbstractDuccComponent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/component/AbstractDuccComponent.java?rev=1847535=1847534=1847535=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/component/AbstractDuccComponent.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/component/AbstractDuccComponent.java
Tue Nov 27 13:31:57 2018
@@ -45,11 +45,11 @@ import org.apache.camel.Exchange;
import org.apache.camel.Processor;
import org.apache.camel.Route;
import org.apache.camel.builder.RouteBuilder;
-import org.apache.camel.model.RouteDefinition;
+import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.admin.event.DuccAdminEvent;
import org.apache.uima.ducc.common.admin.event.DuccAdminEventKill;
-import org.apache.uima.ducc.common.admin.event.DuccAdminEventQuiesceAndStop;
import org.apache.uima.ducc.common.admin.event.DuccAdminEventStop;
+import org.apache.uima.ducc.common.admin.event.TargetableDuccAdminEvent;
import org.apache.uima.ducc.common.crypto
svn commit: r1847489 - /uima/uima-ducc/trunk/src/main/admin/start_ducc
Author: degenaro
Date: Mon Nov 26 21:19:19 2018
New Revision: 1847489
URL: http://svn.apache.org/viewvc?rev=1847489=rev
Log:
UIMA-5875 DUCC admin command stop_ducc should accept --all --stop, which
comprises --agents followed by --head
Skip start of DB if not auto-managed.
> stop_ducc to call ducc_admin for each targeted ducc daemon
Modified:
uima/uima-ducc/trunk/src/main/admin/start_ducc
Modified: uima/uima-ducc/trunk/src/main/admin/start_ducc
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/start_ducc?rev=1847489=1847488=1847489=diff
==
--- uima/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/uima-ducc/trunk/src/main/admin/start_ducc Mon Nov 26 21:19:19 2018
@@ -356,6 +356,10 @@ class StartDucc(DuccUtil):
print sys.exc_info()[0], "Can't start the database."
sys.exit(1)
+if ( len(components) == 1 ):
+if ( self.automanage_database and ('db' in components) ):
+return
+
# activeMQ needs to be started externally before starting any DUCC
processes
if ( self.automanage_broker and ('broker' in components) ):
if ( self.is_amq_active() ):
svn commit: r1844754 - /uima/uima-ducc/trunk/src/main/admin/stop_ducc
Author: degenaro Date: Wed Oct 24 11:37:20 2018 New Revision: 1844754 URL: http://svn.apache.org/viewvc?rev=1844754=rev Log: UIMA-5875 DUCC admin command stop_ducc should accept --all --stop, which comprises --agents followed by --head Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1844754=1844753=1844754=diff == --- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Wed Oct 24 11:37:20 2018 @@ -533,7 +533,7 @@ class StopDucc(DuccUtil): if(component == self.c_broker): stop_broker = True continue -elif(com == self.c_database): +elif(component == self.c_database): stop_db = True continue else:
svn commit: r1844691 - /uima/uima-ducc/trunk/src/main/admin/ducc_post_install
Author: degenaro Date: Tue Oct 23 19:06:45 2018 New Revision: 1844691 URL: http://svn.apache.org/viewvc?rev=1844691=rev Log: UIMA-5899 DUCC ducc_post_install should verify that required /usr/bin/nslookup exists Modified: uima/uima-ducc/trunk/src/main/admin/ducc_post_install Modified: uima/uima-ducc/trunk/src/main/admin/ducc_post_install URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/ducc_post_install?rev=1844691=1844690=1844691=diff == --- uima/uima-ducc/trunk/src/main/admin/ducc_post_install (original) +++ uima/uima-ducc/trunk/src/main/admin/ducc_post_install Tue Oct 23 19:06:45 2018 @@ -424,9 +424,15 @@ class PostInstall(): ret = False return ret - + +def verify_prereqs(self): +fn = '/usr/bin/nslookup' +# will throw exception if not found +dir_stat = os.stat(fn) + def main(self, argv): +self.verify_prereqs() self.DUCC_HOME = find_ducc_home() self.localhost = find_localhost() cwd = os.getcwd()
svn commit: r1843654 - /uima/uima-ducc/trunk/src/main/admin/stop_ducc
Author: degenaro Date: Fri Oct 12 12:10:47 2018 New Revision: 1843654 URL: http://svn.apache.org/viewvc?rev=1843654=rev Log: UIMA-5875 DUCC admin command stop_ducc should accept --all --stop, which comprises --agents followed by --head --all --stop will now stop all daemons listed in DB, including remote head node daemons. However, the command runs slowly since a new JVM is launched to update DB for each daemon. Seems like an opportunity for improvement... Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1843654=1843653=1843654=diff == --- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Fri Oct 12 12:10:47 2018 @@ -229,7 +229,7 @@ class StopDucc(DuccUtil): epilog = epilog+'> '+self._fn()+' '+self.option_component+' '+self.c_or+'@nodeX3'+' '+self.option_kill return epilog -help_all= 'Stop all DUCC management and agent processes by using database entries recorded by start_ducc. Only allowed if '+option_kill+' option is also specified.' +help_all= 'Stop all DUCC management and agent processes by using database entries recorded by start_ducc.' help_head = 'Stop the DUCC management processes on the present head node by using database entries recorded by start_ducc.' help_agents = 'Stop the DUCC agents processes on all nodes by using database entries recorded by '+cmd_start_ducc+'.' help_nodelist = 'Stop agents on the nodes in the nodefile. Multiple nodefiles may be specified.' @@ -239,7 +239,7 @@ class StopDucc(DuccUtil): + ' Specification of broker or database is disallowed unless that component is automanaged by '+kw_DUCC+'.' help_kill = 'Stop the component(s) forcibly and immediately using '+cmd_ssh+' with '+cmd_kill_9+'. Use this only if a normal stop does not work (e.g. the process may be hung).' help_stop = 'Stop the component(s) gracefully using broadcast. Agents allow children specified time (in seconds) to exit. Default is '+str(default_stop)+'.'\ -+ ' Broadcast is not used for broker and database, instead a direct kill -15 is employed.' ++ ' Broadcast is not used for broker, database, and remote head node daemons; instead a direct kill -15 is employed.' help_quiesce= 'Stop the component(s) gracefully using broadcast. Agents exit only when no children exist. Children are given infinite time to exit.' help_maxthreads = 'Maximum concurrent threads. Default = '+str(maxthreads)+'.' help_debug = 'Display debugging messages.' @@ -267,8 +267,9 @@ class StopDucc(DuccUtil): elif(self.args.stop): if(self.args.maxthreads == None): self.args.maxthreads = 2 -else: -self.parser.error(self.option_maxthreads+' requires '+self.option_kill) +elif(self.args.quiesce_then_stop): +if(self.args.maxthreads == None): +self.args.maxthreads = 2 elif(self.args.maxthreads != None): self.parser.error(self.option_maxthreads+' requires '+self.option_kill) # debug @@ -285,6 +286,8 @@ class StopDucc(DuccUtil): def get_db_list(self): if(self.db_list == None): self.db_list = self.db_acct_query() +text = 'list='+str(list) +debug(self._mn(),text) return self.db_list # --all @@ -442,9 +445,13 @@ class StopDucc(DuccUtil): break return pid +def acct_stop(self,node,component): +print 'stop: '+component+'@'+node +self.db_acct_stop(node,component) + # target=kill def kill(self,count,tid,node,component,signal): -self.db_acct_stop(node,component) +self.acct_stop(node,component) verbosity=False ssh = self.ssh_operational(node,verbosity) state = 'state=pending' @@ -472,7 +479,7 @@ class StopDucc(DuccUtil): self.pool.release() # launch threads to perform kills -def kill_threads(self,list): +def kill_threads(self,list,signal): size = len(list) msg = 'daemons='+str(len(list)) output(msg) @@ -486,25 +493,42 @@ class StopDucc(DuccUtil): count = count+1 self.pool.acquire() tid = self.get_tid() -signal = self.sig9 t = Thread(target=self.kill, args=(count,tid,node,component,signal)) t.start() - + +def filter_remote_head(self,list): +list_remote_head = [] +list_remainder = [] +this_node = self.get_node_name() +for i
svn commit: r1843438 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server: DuccHandlerClassic.java DuccHandlerJsonFormat.java
Author: degenaro Date: Wed Oct 10 12:27:44 2018 New Revision: 1843438 URL: http://svn.apache.org/viewvc?rev=1843438=rev Log: UIMA-5890 DUCC Web Server (WS) displays old boot date for re-cycled Broker Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1843438=1843437=1843438=diff == --- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java (original) +++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java Wed Oct 10 12:27:44 2018 @@ -100,7 +100,6 @@ public class DuccHandlerClassic extends private static Messages messages = Messages.getInstance(); private static DuccId jobid = null; - private static BrokerHelper brokerHelper = BrokerHelper.getInstance(); private static DatabaseHelper databaseHelper = DatabaseHelper.getInstance(); private static INodeState nodeState = NodeState.getInstance(); @@ -1443,6 +1442,8 @@ public class DuccHandlerClassic extends DuccDaemonsData duccDaemonsData = DuccDaemonsData.getInstance(); DuccMachinesData duccMachinesData = DuccMachinesData.getInstance(); + BrokerHelper brokerHelper = BrokerHelper.getInstance(); + int counter = 0; boolean brokerAlive = brokerHelper.isAlive(); daemons: Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1843438=1843437=1843438=diff == --- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original) +++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Wed Oct 10 12:27:44 2018 @@ -116,7 +116,6 @@ public class DuccHandlerJsonFormat exten private static Messages messages = Messages.getInstance(); private static DuccId jobid = null; - private static BrokerHelper brokerHelper = BrokerHelper.getInstance(); private static DatabaseHelper databaseHelper = DatabaseHelper.getInstance(); private HelperSpecifications helperSpecifications = HelperSpecifications.getInstance(); @@ -2270,6 +2269,8 @@ public class DuccHandlerJsonFormat exten DuccDaemonsData duccDaemonsData = DuccDaemonsData.getInstance(); DuccMachinesData duccMachinesData = DuccMachinesData.getInstance(); + BrokerHelper brokerHelper = BrokerHelper.getInstance(); + String wsHostIP = getWebServerHostIP(); String wsHostName = getWebServerHostName(); boolean brokerAlive = brokerHelper.isAlive();
svn commit: r1843012 - in /uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event: AbstractDuccEvent.java AbstractMultiDuccEvent.java DuccJobsStateEvent.java Orche
Author: degenaro
Date: Sat Oct 6 11:52:22 2018
New Revision: 1843012
URL: http://svn.apache.org/viewvc?rev=1843012=rev
Log:
UIMA-5742 Reliable DUCC - Move 2 new reliable DUCC fields out of
AbstractDuccEvent for backwards CLI compatibility
Added:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractMultiDuccEvent.java
(with props)
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractDuccEvent.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/DuccJobsStateEvent.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/OrchestratorStateDuccEvent.java
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractDuccEvent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractDuccEvent.java?rev=1843012=1843011=1843012=diff
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractDuccEvent.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractDuccEvent.java
Sat Oct 6 11:52:22 2018
@@ -18,9 +18,6 @@
*/
package org.apache.uima.ducc.transport.event;
-import java.net.InetAddress;
-
-import org.apache.uima.ducc.common.head.IDuccHead.DuccHeadState;
import org.apache.uima.ducc.transport.event.IDuccContext.DuccContext;
public abstract class AbstractDuccEvent
@@ -37,12 +34,8 @@ implements DuccEvent {
private DuccContext context = DuccContext.Unspecified;
- private DuccHeadState duccHeadState = DuccHeadState.unspecified;
- private String producerHost = "unknown";
-
public AbstractDuccEvent(EventType eventType) {
this.eventType = eventType;
- initProducerHost();
}
public EventType getEventType() {
return eventType;
@@ -81,40 +74,5 @@ implements DuccEvent {
public DuccContext getContext() {
return context;
}
-
- /*
-* DuccHeadState indicates Master or Backup
-*/
- public void setDuccHeadState(DuccHeadState value) {
- if(value != null) {
- duccHeadState = value;
- }
- }
-
- public DuccHeadState getDuccHeadState() {
- DuccHeadState retVal = DuccHeadState.unspecified;
- if(duccHeadState != null) {
- retVal = duccHeadState;
- }
- return retVal;
- }
-
- private void initProducerHost() {
- try {
- String host =
InetAddress.getLocalHost().getCanonicalHostName();
- if(host != null) {
- host = host.trim();
- if(host.length() > 0) {
- producerHost = host;
- }
- }
- }
- catch(Exception e) {
- e.printStackTrace();
- }
- }
-
- public String getProducerHost() {
- return producerHost;
- }
+
}
Added:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractMultiDuccEvent.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractMultiDuccEvent.java?rev=1843012=auto
==
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractMultiDuccEvent.java
(added)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/AbstractMultiDuccEvent.java
Sat Oct 6 11:52:22 2018
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language gove
svn commit: r1842893 - in /uima/uima-ducc/trunk: uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/ uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ ui
Author: degenaro
Date: Fri Oct 5 11:36:32 2018
New Revision: 1842893
URL: http://svn.apache.org/viewvc?rev=1842893=rev
Log:
UIMA-5883 DUCC JobDriver (JD) may cause job to never process all work items if
JobProcess (JP) is preempted
Modified:
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionGet.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
Modified:
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java?rev=1842893=1842892=1842893=diff
==
---
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
Fri Oct 5 11:36:32 2018
@@ -122,6 +122,7 @@ public class Standardize {
skewCount,
runMax,
runMin,
+ userKey,
;
Label() {
Modified:
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionGet.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionGet.java?rev=1842893=1842892=1842893=diff
==
---
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionGet.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionGet.java
Fri Oct 5 11:36:32 2018
@@ -37,6 +37,7 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.JobDriverHelper;
import org.apache.uima.ducc.container.jd.blacklist.JobProcessBlacklist;
import org.apache.uima.ducc.container.jd.cas.CasManager;
+import org.apache.uima.ducc.container.jd.cas.CasManagerStats.RetryReason;
import org.apache.uima.ducc.container.jd.log.LoggerHelper;
import org.apache.uima.ducc.container.jd.mh.RemoteWorkerProcess;
import
org.apache.uima.ducc.container.jd.mh.iface.IOperatingInfo.CompletionType;
@@ -161,6 +162,25 @@ public class ActionGet implements IActio
return mmc;
}
+ private synchronized void ungetMetaMetaCas(IActionData actionData,
IRemoteWorkerProcess rwp, IMetaMetaCas mmc, RetryReason rr) throws
JobDriverException {
+ String location = "ungetMetaMetaCas";
+ JobDriver jd = JobDriver.getInstance();
+ CasManager cm = jd.getCasManager();
+ if(mmc != null) {
+ IMetaTask metaCas = mmc.getMetaCas();
+ if(metaCas != null) {
+ String wiId = metaCas.getUserKey();
+ MessageBuffer mb =
LoggerHelper.getMessageBuffer(actionData);
+
mb.append(Standardize.Label.node.get()+rwp.getNodeName());
+
mb.append(Standardize.Label.pid.get()+rwp.getPid());
+ mb.append(Standardize.Label.userKey+wiId);
+ logger.warn(location, ILogger.null_id,
mb.toString());
+ mmc.setMetaCas(null);
+ cm.putMetaCas(metaCas, rr);
+ }
+ }
+ }
+
@Override
public void engage(Object objectData) {
String location = "engage";
@@ -183,7 +203,7 @@ public class ActionGet implements IActio
IMetaMetaCas mmc = getMetaMetaCas(actionData);
if(mmc.isExhausted()) {
Long time =
warnedExhausted.putIfAbsent(rwp, new Long(System.currentTimeMillis()));
- if(time != null) {
+ if(time == null) {
MessageBuffer mbx =
LoggerHelper.getMessageBuffer(actionData);
mbx.append(Standardize.Label.node.get()+rwp.getNodeName());
mbx.append(Standardize.Label.pid.get()+rwp.getPid());
@@ -194,7 +214,7 @@ public class ActionGet implements IActio
}
if(mmc.isPremature()) {
Long time =
warnedPrema
svn commit: r1842843 - /uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Author: degenaro
Date: Thu Oct 4 18:58:35 2018
New Revision: 1842843
URL: http://svn.apache.org/viewvc?rev=1842843=rev
Log:
UIMA-5880 DUCC Web Server (WS) display of Tasks column on Job Performance tab
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1842843=1842842=1842843=diff
==
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Thu Oct 4 18:58:35 2018
@@ -1504,7 +1504,7 @@ public class DuccHandler extends DuccAbs
duccLogger.debug(location, jobid,
"map.size="+map.size());
for (Entry entry : map.entrySet()) {
PerformanceMetricsSummaryItem item =
entry.getValue();
- duccLogger.debug(location, jobid,
"displayName=", item.getDisplayName(), "analysisTime=", item.getAnalysisTime(),
"analysisTimeMin=", item.getAnalysisTimeMin(), "analysisTimeMax=",
item.getAnalysisTimeMax(), "NumProcessed=", item.getNumProcessed());
+ duccLogger.debug(location, jobid,
"displayName=", item.getDisplayName(), "analysisTime=", item.getAnalysisTime(),
"analysisTimeMin=", item.getAnalysisTimeMin(), "analysisTimeMax=",
item.getAnalysisTimeMax(), "analysisTasks=", item.getAnalysisTasks());
}
}
}
@@ -1549,7 +1549,7 @@ public class DuccHandler extends DuccAbs
long anTime = item.getAnalysisTime();
long anMinTime =
item.getAnalysisTimeMin();
long anMaxTime =
item.getAnalysisTimeMax();
- long anTasks = item.getNumProcessed();
+ long anTasks = item.getAnalysisTasks();
analysisTime += anTime;
UimaStatistic stat = new
UimaStatistic(shortname, entry.getKey(), anTime, anMinTime, anMaxTime, anTasks);
uimaStats.add(stat);
svn commit: r1842752 - in /uima/uima-ducc/trunk: uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ uima-duc
Author: degenaro
Date: Wed Oct 3 19:26:47 2018
New Revision: 1842752
URL: http://svn.apache.org/viewvc?rev=1842752=rev
Log:
UIMA-5880 DUCC Web Server (WS) display of Tasks column on Job Performance tab
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/IJobPerformanceSummary.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/JobPerformanceSummary.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceIndividualInfo.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceIndividualKeeper.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceSummaryInfo.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceSummaryKeeper.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceIndividualInfo.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceIndividualKeeper.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceSummaryInfo.java
uima/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceSummaryKeeper.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/job.details.table.performance.jsp
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/IJobPerformanceSummary.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/IJobPerformanceSummary.java?rev=1842752=1842751=1842752=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/IJobPerformanceSummary.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/IJobPerformanceSummary.java
Wed Oct 3 19:26:47 2018
@@ -35,6 +35,9 @@ public interface IJobPerformanceSummary
public long getAnalysisTimeMax();
public void setAnalysisTimeMax(long value);
+ public long getAnalysisTasks();
+ public void setAnalysisTasks(long value);
+
public long getNumProcessed();
public void setNumProcessed(long value);
}
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/JobPerformanceSummary.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/JobPerformanceSummary.java?rev=1842752=1842751=1842752=diff
==
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/JobPerformanceSummary.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/JobPerformanceSummary.java
Wed Oct 3 19:26:47 2018
@@ -30,7 +30,8 @@ public class JobPerformanceSummary imple
private AtomicLong analysisTimeMin = new AtomicLong(-1);
private AtomicLong analysisTimeMax = new AtomicLong(-1);
-
+
+ private AtomicLong analysisTasks = new AtomicLong(0);
public String getName() {
return name;
@@ -78,7 +79,17 @@ public class JobPerformanceSummary imple
public void setAnalysisTimeMax(long value) {
analysisTimeMax.set(value);
}
+
+
+ public long getAnalysisTasks() {
+ return analysisTasks.get();
+ }
+
+ public void setAnalysisTasks(long value) {
+ analysisTasks.set(value);
+ }
+
public long getNumProcessed() {
return numProcessed.get();
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache
svn commit: r1842561 - /uima/uima-ducc/trunk/src/main/admin/stop_ducc
Author: degenaro Date: Mon Oct 1 20:19:22 2018 New Revision: 1842561 URL: http://svn.apache.org/viewvc?rev=1842561=rev Log: UIMA-5742 Reliable DUCC - fix UnboundLocalError: local variable 'process' referenced before assignment Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc Modified: uima/uima-ducc/trunk/src/main/admin/stop_ducc URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/src/main/admin/stop_ducc?rev=1842561=1842560=1842561=diff == --- uima/uima-ducc/trunk/src/main/admin/stop_ducc (original) +++ uima/uima-ducc/trunk/src/main/admin/stop_ducc Mon Oct 1 20:19:22 2018 @@ -451,8 +451,8 @@ class StopDucc(DuccUtil): pfx = 'kill'+' '+'daemon='+str(count)+' '+'thread='+str(tid)+' '+'node='+node+' '+'component='+component+' ' msg = pfx+state output(msg) +process='' if(ssh): -process='' state='state=success' status, tuples = self.find_ducc_process(node) if(status):
svn commit: r1840600 - in /uima/uima-ducc/trunk: ./ uima-ducc-orchestrator/
Author: degenaro Date: Tue Sep 11 18:53:01 2018 New Revision: 1840600 URL: http://svn.apache.org/viewvc?rev=1840600=rev Log: UIMA-5742 Reliable DUCC - svn ignore .pydevproject Modified: uima/uima-ducc/trunk/ (props changed) uima/uima-ducc/trunk/uima-ducc-orchestrator/ (props changed) Propchange: uima/uima-ducc/trunk/ -- --- svn:ignore (original) +++ svn:ignore Tue Sep 11 18:53:01 2018 @@ -2,3 +2,4 @@ target .project issuesFixed +.pydevproject Propchange: uima/uima-ducc/trunk/uima-ducc-orchestrator/ -- --- svn:ignore (original) +++ svn:ignore Tue Sep 11 18:53:01 2018 @@ -2,3 +2,4 @@ target .classpath .project +.pydevproject
