Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/runducc Tue Sep 15 14:31:24 2015 @@ -70,7 +70,10 @@ class DuccProcess(Thread): for line in f: toks = line.strip().split('='); if ( toks[0].strip() == 'threads' ): - threads = toks[1] + if ( self.runner.thread_override == None ): + threads = toks[1] + else: + threads = self.runner.thread_override elif (toks[0].strip() == 'class'): clz = toks[1] elif (toks[0].strip() == 'memory'): @@ -555,12 +558,15 @@ class RunDucc(DuccUtil): print ' --init_fail_cap number-of-failures.' print ' This is the max init failures tolerated before the system starts to cap processes. Default:', self.init_fail_cap print '' - print ' -IB' + print ' --IB' print ' The JP will leak in init() until DUCC (hopefully) kills us' print '' - print ' -PB' + print ' --PB' print ' The JP will leak in process() until DUCC (hopefully) kills us' print '' + print ' --TO number-of-threads' + print ' Thread-override: force this number of threads regardless of what is in job spec.' + print '' print ' -r, --range seconds' print ' This is the AE initializion time range over base in milliseconds. Default:', self.init_range print ' Init time is -i value + random[0, -rvalue]' @@ -643,13 +649,14 @@ class RunDucc(DuccUtil): self.max_machines = 0 self.use_http = False self.descriptor_as_file = False + self.thread_override = None try: opts, args = getopt.getopt(argv, 'b:d:fi:m:n:op:q:r:s:t:u:v:w:x:y:z:?h', ['AE', 'DD', 'file', 'SE=', 'IB=', 'PB=', 'directory=', 'batchfile=', 'init_time=', 'init_fail_cap=', 'range=', 'memory_override=', 'nmachines=', 'process_timeout=', 'init_timeout=', 'observe' 'jd_uima_log=', 'jp_uima_log=', - 'http', + 'http', 'threads=' ]) except: print "Unknown option" @@ -698,6 +705,8 @@ class RunDucc(DuccUtil): self.init_bloat = a elif o in ('--PB'): self.process_bloat = a + elif o in ('--threads'): + self.thread_override = a elif ( o == '-q'): self.cr_getnext_bloat = int(a) elif ( o == '-s'): @@ -752,6 +761,7 @@ class RunDucc(DuccUtil): print ' jd_uima_log :', self.jd_uima_log print ' jp_uima_log :', self.jp_uima_log print ' DUCC_HOME :', self.DUCC_HOME + print ' Thread override :', self.thread_override self.submit_package = 'org.apache.uima.ducc'
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/start_sim Tue Sep 15 14:31:24 2015 @@ -44,6 +44,10 @@ from properties import Properties from ducc import Ducc from ducc_util import ThreadPool +# multi-threaded start can overwhelm ssh if going to the same target host. We inject +# a short sleep between starts to make it better. This is how long to sleep. +global SLEEP_TIME +SLEEP_TIME = .1 class StartSim(DuccUtil): @@ -58,17 +62,6 @@ class StartSim(DuccUtil): broker_host = self.ducc_properties.get('ducc.broker.hostname') print 'broker host', broker_host lines = self.ssh(broker_host, True, "'", self.DUCC_HOME + '/admin/ducc.py', '-c', 'broker', "'") - while 1: - line = lines.readline().strip() - if ( not line ): - break - #print '[] ' + line - if ( line.startswith('PID') ): - toks = line.split(' ') # get the PID - print "Broker on", broker_host, 'PID', toks[1] - self.pids.put('broker@' + broker_host, toks[1]) - lines.close() - break for i in range(0, 9): if ( self.is_amq_active() ): @@ -128,7 +121,7 @@ class StartSim(DuccUtil): line = lines.readline().strip() if ( not line ): break - # print '[] ' + line + #print '[] ' + line if ( line.startswith('PID') ): toks = line.split(' ') # get the PID self.pidlock.acquire(); @@ -226,7 +219,7 @@ class StartSim(DuccUtil): ip = '192.168.4.' + index pnode = node + '-' + index self.threadpool.invoke(self.startOneAgent, node, cmd, mem, ip, pnode, index) - time.sleep(.1) + time.sleep(SLEEP_TIME) def usage(self, msg): if (msg != None): @@ -383,15 +376,34 @@ class StartSim(DuccUtil): self.invalid("Running with a nodelist is not compatible with running a single agent."); try: + specials = ['broker', 'db'] + sc = set(components) + sb = set(specials) + specials_only = False + if ( sc.issubset(sb) ): + read_pids = True + + print '-------- start broker' if ( components.get('broker') != None ): self.start_broker() + print '-------- start database' + if ( components.get('db') != None ): + if ( self.db_start() ): + self.db_init() + + print '-------- specials_only', specials_only + if ( specials_only ): + return + + print '-------- start agents' if ( node_config != None ): self.startAgents(node_config, instances) for (com, com) in components.items(): - if ( com != 'broker' ): # started separately + if ( not com in specials ): # specials start with different rules self.threadpool.invoke(self.startComponent, com, or_parms) + time.sleep(SLEEP_TIME) except: pass Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/scripts/stop_sim Tue Sep 15 14:31:24 2015 @@ -37,6 +37,36 @@ from ducc_util import ThreadPool class StopSim(DuccUtil): + def stop_db(self): + + # disable until we're ready to turn this on + if ( self.db_parms == self.db_disabled ): + return True + + (jvm_parms, classpath, db_rt, dburl, dbroot) = self.db_parms() + + print 'Stopping database' + + main = 'com.orientechnologies.orient.server.OServerShutdownMain' + + jp = '' + for k in jvm_parms.keys(): + v = jvm_parms[k] + if ( v == None ): + jp = jp + k + ' ' + else: + jp = jp + k + '=' + v + ' ' + + + cmd = ' '.join([self.java(), jp, '-cp', classpath, main]) + print cmd + + here = os.getcwd() + os.chdir(db_rt) + self.spawn(cmd) + os.chdir(here) + + def signal_process(self, args): inst, data_prop, signal = args data = data_prop @@ -103,6 +133,7 @@ class StopSim(DuccUtil): instances = [] signal = '-INT' stopbroker = False + stopdatabase = False try: opts, args = getopt.getopt(argv, 'c:kprvh?', ['component=', 'kill' 'pause', 'resume', 'nothreading', 'version', 'help']) @@ -143,15 +174,19 @@ class StopSim(DuccUtil): if ( signal in ('-KILL', '-INT') ): pids.delete(inst) stopbroker = True + stopdatabase = True else: for inst in instances: + print 'Instance', inst data = pids.get(inst) if ( data == None ): if ( inst == 'broker' and signal in ('-KILL', '-INT') ): stopbroker = True else: print "WARNING - No PID found for instance", inst + elif ( inst == 'db' ): + stopdatabase = True else: self.threadpool.invoke(self.signal_process, inst, pids.get(inst), signal) time.sleep(.1) @@ -170,6 +205,9 @@ class StopSim(DuccUtil): print "Waiting " + str(sleeptime) + " seconds to broadcast agent shutdown." time.sleep(sleeptime) + if ( stopdatabase ) : + self.db_stop() + if ( stopbroker and self.automanage ): print "Stopping broker" self.stop_broker() Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/pom.xml Tue Sep 15 14:31:24 2015 @@ -62,6 +62,11 @@ </dependency> <dependency> + <groupId>org.apache.uima</groupId> + <artifactId>uima-ducc-database</artifactId> + </dependency> + + <dependency> <groupId>org.apache.camel</groupId> <artifactId>camel-jetty</artifactId> </dependency> Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java Tue Sep 15 14:31:24 2015 @@ -24,14 +24,21 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.uima.ducc.common.internationalization.Messages; import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccLoggerComponents; import org.apache.uima.ducc.common.utils.IOHelper; +import org.apache.uima.ducc.common.utils.id.DuccId; import org.apache.uima.ducc.orchestrator.utilities.Checkpointable; import org.apache.uima.ducc.orchestrator.utilities.TrackSync; import org.apache.uima.ducc.transport.event.common.DuccWorkMap; +import org.apache.uima.ducc.transport.event.common.DuccWorkReservation; +import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType; +import org.apache.uima.ducc.transport.event.common.history.HistoryFactory; +import org.apache.uima.ducc.transport.event.common.history.IHistoryPersistenceManager; public class OrchestratorCheckpoint { @@ -44,8 +51,12 @@ public class OrchestratorCheckpoint { private static String fileName = orchestratorCommonArea.getStateDirectory()+File.separator+"orchestrator.ckpt"; private static OrchestratorCheckpoint orchestratorCheckpoint = new OrchestratorCheckpoint(); - - public static OrchestratorCheckpoint getInstance() { + + private static boolean useDb = true; + + public static OrchestratorCheckpoint getInstance() + { + useDb = System.getProperty("ducc.job.history.impl").contains("database"); return orchestratorCheckpoint; } @@ -140,9 +151,42 @@ public class OrchestratorCheckpoint { logger.trace(methodName, null, messages.fetch("exit")); return; } - - public boolean saveState() { - String methodName = "saveState"; + + private boolean saveStateDb() + { + String methodName = "saveStateDb"; + IHistoryPersistenceManager saver = HistoryFactory.getInstance(this.getClass().getName()); + logger.trace(methodName, null, messages.fetch("enter")); + boolean retVal = false; + if(saveEnabled) { + DuccWorkMap workMap = orchestratorCommonArea.getWorkMap(); + TrackSync ts = TrackSync.await(workMap, this.getClass(), methodName); + synchronized(workMap) { + ts.using(); + try + { + logger.info(methodName, null, messages.fetchLabel("saving to")+fileName); + Checkpointable checkpointable = orchestratorCommonArea.getCheckpointable(); + retVal = saver.checkpoint(checkpointable.getWorkMap(), checkpointable.getProcessToJobMap()); + logger.info(methodName, null, messages.fetchLabel("saved")+fileName); + } + catch(Exception e) + { + logger.error(methodName, null, e); + } + } + ts.ended(); + } + else { + logger.debug(methodName, null, messages.fetchLabel("bypass saving to")+fileName); + } + logger.trace(methodName, null, messages.fetch("exit")); + return retVal; + } + + private boolean saveStateFile() + { + String methodName = "saveStateFile"; logger.trace(methodName, null, messages.fetch("enter")); boolean retVal = false; if(saveEnabled) { @@ -175,9 +219,57 @@ public class OrchestratorCheckpoint { } logger.trace(methodName, null, messages.fetch("exit")); return retVal; + } + + public boolean saveState() + { + // we can resolve these into just one call by allowing the checkpointable to be saved in its parts for + // the file implementation, to avoid circular dependencies + if ( useDb ) return saveStateDb(); + else return saveStateFile(); } - - public boolean restoreState() { + + private boolean restoreStateDb() + { + String methodName = "restoreState"; + logger.trace(methodName, null, messages.fetch("enter")); + IHistoryPersistenceManager saver = HistoryFactory.getInstance(this.getClass().getName()); + boolean retVal = false; + if(saveEnabled) { + DuccWorkMap workMap = orchestratorCommonArea.getWorkMap(); + TrackSync ts = TrackSync.await(workMap, this.getClass(), methodName); + synchronized(workMap) { + ts.using(); + try + { + logger.info(methodName, null, messages.fetchLabel("restoring from")+fileName); + DuccWorkMap work = new DuccWorkMap(); + ConcurrentHashMap<DuccId, DuccId> processToJob = new ConcurrentHashMap<DuccId, DuccId>(); + Checkpointable checkpointable = new Checkpointable(work, processToJob); + retVal = saver.restore(work, processToJob); + orchestratorCommonArea.setCheckpointable(checkpointable); + logger.info(methodName, null, messages.fetch("restored")); + } + catch(ClassNotFoundException e) + { + logger.error(methodName, null, e); + } + + catch(Exception e) + { + logger.warn(methodName, null, e); + } + } + ts.ended(); + } + else { + logger.info(methodName, null, messages.fetchLabel("bypass restoring from")+fileName); + } + logger.trace(methodName, null, messages.fetch("exit")); + return retVal; + } + + private boolean restoreStateFile() { String methodName = "restoreState"; logger.trace(methodName, null, messages.fetch("enter")); boolean retVal = false; @@ -194,6 +286,15 @@ public class OrchestratorCheckpoint { fis = new FileInputStream(fileName); in = new ObjectInputStream(fis); Checkpointable checkpointable = (Checkpointable)in.readObject(); + DuccWorkMap map = checkpointable.getWorkMap(); + + Set<DuccId> ids = map.getReservationKeySet(); + for ( DuccId id : ids ) { + DuccWorkReservation r = (DuccWorkReservation) map.findDuccWork(DuccType.Reservation, ""+id.getFriendly()); + logger.info(methodName, id, "Looking for work: r", r); + if ( r != null ) r.initLogger(); + } + orchestratorCommonArea.setCheckpointable(checkpointable); in.close(); retVal = true; @@ -217,4 +318,13 @@ public class OrchestratorCheckpoint { return retVal; } + public boolean restoreState() + { + // we can resolve these into just one call by allowing the checkpointable to be saved in its parts for + // the file implementation, to avoid circular dependencies + + if ( useDb ) return restoreStateDb(); + else return restoreStateFile(); + } + } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java Tue Sep 15 14:31:24 2015 @@ -119,6 +119,7 @@ public class OrchestratorCommonArea { private void init() { // <Jira 3414> + String methodName="init"; DuccPropertiesResolver dpr = DuccPropertiesResolver.getInstance(); Boolean use_lock_file = new Boolean(dpr.getProperty(DuccPropertiesResolver.ducc_orchestrator_use_lock_file)); if(use_lock_file) { @@ -133,7 +134,13 @@ public class OrchestratorCommonArea { OrchestratorCheckpoint.getInstance().switchOnOff(commonConfiguration.orchestratorCheckpoint); OrchestratorCheckpoint.getInstance().restoreState(); jdScheduler = JdScheduler.getInstance(); - historyPersistenceManager = HistoryFactory.getInstance(); + try { + historyPersistenceManager = HistoryFactory.getInstance(this.getClass().getName()); + } catch ( Exception e ) { + logger.error(methodName, null, "Cannot acquire the history manager", e); + System.exit(1); // what should we do here? exit or acquire the NullHistoryManager? + } + logger.info(methodName, null, "Got history manager of class", historyPersistenceManager.getClass().getName()); } public String getStateDirectory() { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java Tue Sep 15 14:31:24 2015 @@ -203,11 +203,11 @@ public class StateManager { try { switch(duccWorkJob.getDuccType()) { case Job: - hpm.jobSave(duccWorkJob); + hpm.saveJob(duccWorkJob); retVal = true; break; case Service: - hpm.serviceSave((IDuccWorkService)duccWorkJob); + hpm.saveService((IDuccWorkService)duccWorkJob); retVal = true; break; } @@ -224,7 +224,7 @@ public class StateManager { logger.trace(methodName, null, messages.fetch("enter")); boolean retVal = false; try { - hpm.reservationSave(duccWorkReservation); + hpm.saveReservation(duccWorkReservation); retVal = true; } catch(Exception e) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/utilities/Checkpointable.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/utilities/Checkpointable.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/utilities/Checkpointable.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/utilities/Checkpointable.java Tue Sep 15 14:31:24 2015 @@ -33,7 +33,7 @@ public class Checkpointable implements S public Checkpointable(DuccWorkMap workMap, ConcurrentHashMap<DuccId,DuccId> processToJobMap) { this.workMap = workMap; - this.processToJobMap = processToJobMap; + this.processToJobMap = processToJobMap; } public DuccWorkMap getWorkMap() { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml Tue Sep 15 14:31:24 2015 @@ -165,6 +165,8 @@ <jetty.version>7.4.4.v20110707</jetty.version> <orbit-org-apache-jasper.version>2.1.0.v201110031002</orbit-org-apache-jasper.version> <servlet-api.version>2.5</servlet-api.version> + <orientdb.version>2.1.0</orientdb.version> + <orientdb.studio.version>2.0-M3</orientdb.studio.version> <http.commons.client.version>4.3.5</http.commons.client.version> <http.commons.client-cache.version>4.3.5</http.commons.client-cache.version> @@ -269,6 +271,12 @@ ${uimaDUCCNoticeText} <dependency> <groupId>org.apache.uima</groupId> + <artifactId>uima-ducc-database</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.uima</groupId> <artifactId>uima-ducc-transport</artifactId> <version>${project.version}</version> </dependency> @@ -753,6 +761,7 @@ ${uimaDUCCNoticeText} <exclude>uima-ducc-sm/**</exclude> <exclude>uima-ducc-spawn/**</exclude> <exclude>uima-ducc-transport/**</exclude> + <exclude>uima-ducc-database/**</exclude> <exclude>uima-ducc-web/**</exclude> <exclude>uima-ducc-user/**</exclude> <exclude>uima-ducc-container/**</exclude> Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java Tue Sep 15 14:31:24 2015 @@ -18,7 +18,6 @@ */ package org.apache.uima.ducc.sm; -import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; @@ -29,6 +28,7 @@ import java.util.Set; import org.apache.uima.ducc.cli.DuccServiceApi; import org.apache.uima.ducc.cli.IUiOptions.UiOption; import org.apache.uima.ducc.common.NodeIdentity; +import org.apache.uima.ducc.common.persistence.services.IStateServices; import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccProperties; import org.apache.uima.ducc.common.utils.id.DuccId; @@ -68,6 +68,8 @@ public class ServiceHandler private ServiceStateHandler serviceStateHandler = new ServiceStateHandler(); private ServiceMap serviceMap = new ServiceMap(); // note this is the sync object for publish + private IStateServices stateHandler; + private Map<DuccId, IDuccWork> newJobs = new HashMap<DuccId, IDuccWork>(); private Map<DuccId, IDuccWork> newServices = new HashMap<DuccId, IDuccWork>(); @@ -96,6 +98,11 @@ public class ServiceHandler } } + void setStateHandler(IStateServices handler) + { + this.stateHandler = handler; + } + public synchronized void run() { String methodName = "run"; @@ -120,6 +127,7 @@ public class ServiceHandler */ void bootImplementors(Map<DuccId, DuccWorkJob> incoming) { + String methodName = "bootImplementors"; for ( DuccId id : incoming.keySet() ) { DuccWorkJob j = incoming.get(id); String ep = j.getServiceEndpoint(); @@ -133,7 +141,11 @@ public class ServiceHandler } List<ServiceSet> services = serviceStateHandler.getServices(); for ( ServiceSet sset : services ) { - sset.bootComplete(); + try { + sset.bootComplete(); + } catch ( Exception e ) { + logger.warn(methodName, sset.getId(), "Error updating meta properties:", e); + } if ( sset.countImplementors() > 0 ) { // if something was running, let's make sure all the starts are done sset.start(); } @@ -884,6 +896,7 @@ public class ServiceHandler synchronized ServiceReplyEvent disable(ServiceDisableEvent ev) { + String methodName = "disable"; long id = ev.getFriendly(); String url = ev.getEndpoint(); ServiceSet sset = serviceStateHandler.getServiceForApi(id, url); @@ -900,12 +913,18 @@ public class ServiceHandler } sset.disable("Disabled by owner or administrator " + ev.getUser()); - sset.saveMetaProperties(); + try { + sset.updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, sset.getId(), "Error updating meta properties:", e); + } + return ServiceManagerComponent.makeResponse(true, "Disabled", sset.getKey(), sset.getId().getFriendly()); } synchronized ServiceReplyEvent enable(ServiceEnableEvent ev) { + String methodName = "enable"; long id = ev.getFriendly(); String url = ev.getEndpoint(); ServiceSet sset = serviceStateHandler.getServiceForApi(id, url); @@ -922,7 +941,11 @@ public class ServiceHandler } sset.enable(); - sset.saveMetaProperties(); + try { + sset.updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, sset.getId(), "Error updating meta properties:", e); + } return ServiceManagerComponent.makeResponse(true, "Enabled.", sset.getKey(), sset.getId().getFriendly()); } @@ -981,7 +1004,7 @@ public class ServiceHandler return ServiceManagerComponent.makeResponse(true, "Observing references.", sset.getKey(), sset.getId().getFriendly()); } - synchronized ServiceReplyEvent register(DuccId id, String props_filename, String meta_filename, DuccProperties props, DuccProperties meta) + synchronized ServiceReplyEvent register(DuccId id, DuccProperties props, DuccProperties meta, boolean isRecovered) { String methodName = "register"; @@ -996,7 +1019,7 @@ public class ServiceHandler } try { - sset = new ServiceSet(this, id, props_filename, meta_filename, props, meta); + sset = new ServiceSet(this, this.stateHandler, id, props, meta); } catch (Throwable t) { // throws because endpoint is not parsable error = t.getMessage(); @@ -1004,22 +1027,14 @@ public class ServiceHandler } try { - sset.saveServiceProperties(); + // if it's a "fresh" reservation it must go into the db. otherwise it is already + // in the db and doesn't need to be inserted + sset.storeProperties(isRecovered); } catch ( Exception e ) { error = ("Internal error; unable to store service descriptor. " + url); logger.error(methodName, id, e); - must_deregister = true; } - try { - if ( ! must_deregister ) { - sset.saveMetaProperties(); - } - } catch ( Exception e ) { - error = ("Internal error; unable to store service meta-descriptor. " + url); - logger.error(methodName, id, e); - must_deregister = true; - } // must check for cycles or we can deadlock if ( ! must_deregister ) { @@ -1036,16 +1051,11 @@ public class ServiceHandler serviceStateHandler.registerService(id.getFriendly(), url, sset); return ServiceManagerComponent.makeResponse(true, "Registered", url, id.getFriendly()); } else { - File mf = new File(meta_filename); - mf.delete(); - - File pf = new File(props_filename); - pf.delete(); return ServiceManagerComponent.makeResponse(false, error, url, id.getFriendly()); } } - synchronized public ServiceReplyEvent modify(ServiceModifyEvent ev) + synchronized ServiceReplyEvent modify(ServiceModifyEvent ev) { long id = ev.getFriendly(); String url = ev.getEndpoint(); @@ -1059,7 +1069,7 @@ public class ServiceHandler } pendingRequests.add(new ApiHandler(ev, this)); - return ServiceManagerComponent.makeResponse(true, "Modifying", sset.getKey(), sset.getId().getFriendly()); + return ServiceManagerComponent.makeResponse(true, "Modify accepted:", sset.getKey(), sset.getId().getFriendly()); } boolean restart_pinger = false; @@ -1189,8 +1199,11 @@ public class ServiceHandler } sset.resetRuntimeErrors(); - sset.saveServiceProperties(); - sset.saveMetaProperties(); + try { + sset.updateSvcProperties(); + } catch (Exception e) { + logger.error(methodName, sset.getId(), "Cannot store properties:", e); + } if ( restart_pinger ) { sset.restartPinger(); @@ -1200,7 +1213,7 @@ public class ServiceHandler // restart_service - not yet } - synchronized public ServiceReplyEvent unregister(ServiceUnregisterEvent ev) + synchronized ServiceReplyEvent unregister(ServiceUnregisterEvent ev) { //String methodName = "unregister"; long id = ev.getFriendly(); @@ -1244,7 +1257,11 @@ public class ServiceHandler if ( sset.isPingOnly() ) { logger.info(methodName, sset.getId(), "Unregister ping-only setvice:", friendly, url); serviceStateHandler.removeService(sset); - sset.deleteProperties(); + try { + sset.deleteProperties(); + } catch (Exception e) { + logger.error(methodName, sset.getId(), "Cannot delete service from DB:", e); + } } else if ( sset.countImplementors() > 0 ) { logger.debug(methodName, sset.getId(), "Stopping implementors:", friendly, url); } else { @@ -1410,14 +1427,21 @@ public class ServiceHandler { ServiceShutdown() { + System.out.println("Setting shutdown hook"); } public void run() { + System.out.println("Running shutdown hook"); List<ServiceSet> allServices = serviceStateHandler.getServices(); for (ServiceSet sset : allServices) { sset.stopMonitor(); } + try { + stateHandler.shutdown(); + } catch ( Exception e ) { + logger.warn("ServicShutdown.run", null, "Error closing database: ", e); + } } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java Tue Sep 15 14:31:24 2015 @@ -22,8 +22,11 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; import java.util.Map; +import org.apache.uima.ducc.common.persistence.services.IStateServices; import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccProperties; import org.apache.uima.ducc.transport.event.common.IDuccState.JobState; @@ -162,31 +165,46 @@ class ServiceInstance // this.state = dwj.getJobState(); // } - long start(String spec, DuccProperties meta_props) + String[] genArgs(DuccProperties props) + { + List<String> args = new ArrayList<String>(); + + args.add(System.getProperty("ducc.agent.launcher.ducc_spawn_path")); + args.add("-u"); + args.add(user); + args.add("--"); + args.add(System.getProperty("ducc.jvm")); + args.add("-cp"); + args.add(api_classpath); + args.add("org.apache.uima.ducc.cli.DuccServiceSubmit"); + args.add("--service_id"); + args.add(sset.getId().toString()); + + @SuppressWarnings("rawtypes") + Enumeration keys = props.propertyNames(); + while ( keys.hasMoreElements() ) { + String k = (String) keys.nextElement(); + // System.out.println("------ Set argument " + k + " to " + ((String)props.get(k))); + String v = (String) props.get(k); + + args.add("--" + k); + args.add(v); + } + return args.toArray(new String[args.size()]); + } + + long start(DuccProperties svc_props, DuccProperties meta_props) { String methodName = "start"; logger.info(methodName, sset.getId(), "START INSTANCE"); setStopped(false); - this.user = meta_props.getProperty("user"); + this.user = meta_props.getProperty(IStateServices.SvcProps.user.pname()); // Simple use of ducc_ling, just submit as the user. The specification will have the working directory // and classpath needed for the service, handled by the Orchestrator and Job Driver. - String[] args = { - System.getProperty("ducc.agent.launcher.ducc_spawn_path"), - "-u", - user, - "--", - System.getProperty("ducc.jvm"), - "-cp", - api_classpath, - "org.apache.uima.ducc.cli.DuccServiceSubmit", - "--specification", - spec, - "--service_id", - sset.getId().toString(), - }; - + String[] args = genArgs(svc_props); + for ( int i = 0; i < args.length; i++ ) { if ( i > 0 && (args[i-1].equals("-cp") ) ) { // The classpaths can be just awful filling the logs with junk. It will end up in the agent log @@ -228,7 +246,11 @@ class ServiceInstance } catch (Throwable t) { logger.error(methodName, sset.getId(), t); - sset.setErrorString(t.toString()); + try { + sset.setErrorString(t.toString()); + } catch ( Exception e ) { + logger.warn(methodName, sset.getId(), "Error updating meta properties:", e); + } return -1; } @@ -272,7 +294,11 @@ class ServiceInstance started = true; logger.info(methodName, null, "Request to start service " + sset.getId().toString() + " accepted as service instance ", numeric_id); } catch ( NumberFormatException e ) { - sset.setErrorString("Request to start service " + sset.getId().toString() + " failed, can't interpret submit response.: " + s); + try { + sset.setErrorString("Request to start service " + sset.getId().toString() + " failed, can't interpret submit response.: " + s); + } catch ( Exception ee ) { + logger.warn(methodName, sset.getId(), "Error updating meta properties:", ee); + } logger.warn(methodName, null, "Request to start service " + sset.getId().toString() + " failed, can't interpret response.: " + s); } @@ -281,10 +307,10 @@ class ServiceInstance if ( ! started ) { logger.warn(methodName, sset.getId(), "Request to start service " + sset.getId().toString() + " failed."); - meta_props.put("submit-error", submit_buffer.toString()); + meta_props.put(IStateServices.SvcProps.submit_error.pname(), submit_buffer.toString()); sset.log_errors(stdout_lines, stderr_lines); } else { - meta_props.remove("submit-error"); + meta_props.remove(IStateServices.SvcProps.submit_error.pname()); state = JobState.Received; } logger.info(methodName, sset.getId(), "START INSTANCE COMPLETE"); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java Tue Sep 15 14:31:24 2015 @@ -24,6 +24,7 @@ import java.io.FileOutputStream; import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.NavigableSet; import java.util.Properties; import java.util.UUID; @@ -36,6 +37,10 @@ import org.apache.uima.ducc.common.compo import org.apache.uima.ducc.common.crypto.Crypto; import org.apache.uima.ducc.common.crypto.Crypto.AccessType; import org.apache.uima.ducc.common.main.DuccService; +import org.apache.uima.ducc.common.persistence.services.IStateServices; +import org.apache.uima.ducc.common.persistence.services.StateServicesDirectory; +import org.apache.uima.ducc.common.persistence.services.StateServicesFactory; +import org.apache.uima.ducc.common.persistence.services.StateServicesSet; import org.apache.uima.ducc.common.utils.DuccCollectionUtils; import org.apache.uima.ducc.common.utils.DuccCollectionUtils.DuccMapDifference; import org.apache.uima.ducc.common.utils.DuccCollectionUtils.DuccMapValueDifference; @@ -93,6 +98,7 @@ public class ServiceManagerComponent private String stateEndpoint; private ServiceHandler handler = null; + IStateServices stateHandler = null; //HashMap<String, BaseUimaAsService> services = new HashMap<String, BaseUimaAsService>(); @@ -108,7 +114,7 @@ public class ServiceManagerComponent private String state_dir = null; private String state_file = null; - private String descriptor_dir = null; + private DuccProperties sm_props = null; private String service_seqno = "service.seqno"; private DuccIdFactory idFactory = new DuccIdFactory(); @@ -126,7 +132,7 @@ public class ServiceManagerComponent // 1.1.4 - dynamic mod of all registration parms. Add debug and max-init-time parms. // 1.1.0 - resync with release, sigh. // 2.0.0 - Update for new release. - String version = "2.0.0"; + String version = "2.1.0"; public ServiceManagerComponent(CamelContext context) { @@ -139,7 +145,7 @@ public class ServiceManagerComponent { return logger; } - + /** * Initialization tasks: * - read all the service descriptors @@ -149,71 +155,76 @@ public class ServiceManagerComponent throws Exception { String methodName = "init"; - try { - File descdir = new File(serviceFileLocation()); - if ( ! descdir.exists() ) { - descdir.mkdirs(); - } - File histdir = new File(serviceHistoryLocation()); - if ( ! histdir.exists() ) { - histdir.mkdirs(); - } - String[] desclist = descdir.list(); - for ( String d : desclist) { - if ( d.endsWith(".svc") ) { - int ndx = d.lastIndexOf("."); - String stem = d.substring(0, ndx); - - DuccProperties props = new DuccProperties(); - String props_filename = serviceFileKey(d); - props.load(props_filename); - - DuccProperties metaprops = new DuccProperties(); - String meta_filename = serviceFileKey(stem + ".meta"); - metaprops.load(meta_filename); - - int friendly = 0; - String uuid = ""; - try { - // these gets will throw if the requisite objects aren't found - friendly = metaprops.getIntProperty("numeric_id"); - uuid = metaprops.getStringProperty("uuid"); - } catch (MissingPropertyException e1) { - // Ugly, but shouldn't have to be fatal - logger.error(methodName, null, "Cannot restore DuccId for", d, "Friendly id:", friendly, "uuid:", uuid); - continue; - } - - DuccId id = new DuccId(friendly); - id.setUUID(UUID.fromString(uuid)); - logger.debug(methodName, id, "Unique:", id.getUnique()); - - try { - handler.register(id, props_filename, meta_filename, props, metaprops); - } catch (IllegalStateException e ) { // happens on duplicate service - logger.error(methodName, id, e.getMessage()); // message has all I need. - } - - } + + // recover the registry + StateServicesDirectory all = stateHandler.getStateServicesDirectory(); + NavigableSet<Long> svcs = all.getDescendingKeySet(); + + for ( Long l : svcs ) { + StateServicesSet sss = all.get(l); + DuccProperties svcprops = sss.get(IStateServices.svc); + DuccProperties metaprops = sss.get(IStateServices.meta); + + int friendly = 0; + String uuid = ""; + try { + // these gets will throw if the requisite objects aren't found + friendly = metaprops.getIntProperty("numeric_id"); + uuid = metaprops.getStringProperty("uuid"); + } catch (MissingPropertyException e1) { + // Ugly, but shouldn't have to be fatal + logger.error(methodName, null, "Cannot restore DuccId for service", l, "Friendly id:", friendly, "uuid:", uuid); + continue; } + + DuccId id = new DuccId(friendly); + id.setUUID(UUID.fromString(uuid)); + logger.debug(methodName, id, "Unique:", id.getUnique()); + + try { + handler.register(id, svcprops, metaprops, true); + } catch (IllegalStateException e ) { // happens on duplicate service + logger.error(methodName, id, e.getMessage()); // message has all I need. + } + + } - } catch (Throwable e) { - // If we get here we aren't startable. - logger.error(methodName, null, "Cannot initialize service manger: ", e.getMessage()); - System.exit(1); - } + // try { + // File histdir = new File(serviceHistoryLocation()); + // if ( ! histdir.exists() ) { + // histdir.mkdirs(); + // } + + // Map<Long, Properties> sprops = h.getPropertiesForType(DbVertex.Service); + // Map<Long, Properties> mprops = h.getPropertiesForType(DbVertex.ServiceMeta); + + // for ( Long k : sprops.keySet() ) { + // DuccProperties svcprops = (DuccProperties) sprops.get(k); + // DuccProperties metaprops = (DuccProperties) mprops.get(k); + + // String uuid = metaprops.getProperty("uuid"); + + // DuccId id = new DuccId(k); + // id.setUUID(UUID.fromString(uuid)); + // logger.debug(methodName, id, "Unique:", id.getUnique()); + + // try { + // handler.register(id, svcprops, metaprops, true); + // } catch (IllegalStateException e ) { // happens on duplicate service + // logger.error(methodName, id, e); // message has all I need. + // } + // } + + // } catch (Throwable e) { + // // If we get here we aren't startable. + // logger.error(methodName, null, "Cannot initialize service manger: ", e); + // System.exit(1); + // } finally { + // h.close(); + // } state_dir = System.getProperty("DUCC_HOME") + "/state"; state_file = state_dir + "/sm.properties"; - descriptor_dir = state_dir + "/services"; - File ddir = new File(descriptor_dir); - if ( ddir.exists() ) { - if ( ! ddir.isDirectory() ) { - throw new IllegalStateException("Service descriptor location is not a directory: " + descriptor_dir); - } - } else { - ddir.mkdirs(); - } sm_props = new DuccProperties(); File sf = new File(state_file); @@ -331,7 +342,28 @@ public class ServiceManagerComponent logger.info(methodName, null, "------------------------------------------------------------------------------------"); readAdministrators(); + + stateHandler = StateServicesFactory.getInstance(this.getClass().getName(), COMPONENT_NAME); + // // String dbname = System.getProperty("ducc.db.name"); + // String dburl = System.getProperty("ducc.state.database.url"); // "remote:localhost:2424/DuccState" + + // try { + // // verify, and possibly set up the schema if it's the first time + // databaseHandler = new DbManager(dburl); + // databaseHandler.init(); + // } catch (Throwable e) { + // logger.fatal(methodName, null, "Cannot create database at", dburl, ":", e); + // Runtime.getRuntime().halt(1); + // } + + // if ( databaseHandler == null ) { + // logger.error(methodName, null, "Cannot open database at", dburl); + // } else { + // logger.info(methodName, null, "Opened database at", dburl); + // } + handler.setStateHandler(stateHandler); + // Here is a good place to do any pre-start stuff // Start the main processing loop @@ -687,6 +719,7 @@ public class ServiceManagerComponent notify(); } + // @deprecated static String serviceFileLocation() { return System.getProperty("DUCC_HOME") + "/state/services"; @@ -697,11 +730,6 @@ public class ServiceManagerComponent return System.getProperty("DUCC_HOME") + "/history/services-registry/"; } - private String serviceFileKey(String fn) - { - return serviceFileLocation() + "/" + fn; - } - private boolean check_signature(String user, byte[] auth_block) throws Throwable { @@ -807,9 +835,7 @@ public class ServiceManagerComponent meta.setProperty("autostart", "false"); } - String desc_name = descriptor_dir + "/" + id + ".svc"; - String meta_name = descriptor_dir + "/" + id + ".meta"; - ServiceReplyEvent reply = handler.register(id, desc_name, meta_name, props, meta); + ServiceReplyEvent reply = handler.register(id, props, meta, false); ev.setReply(reply); // Draw attentipn in the log on registration failures Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Tue Sep 15 14:31:24 2015 @@ -18,10 +18,6 @@ */ package org.apache.uima.ducc.sm; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.sql.Timestamp; @@ -31,6 +27,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Timer; import java.util.TimerTask; import java.util.TreeMap; @@ -41,6 +38,7 @@ import org.apache.uima.ducc.cli.UimaAsPi import org.apache.uima.ducc.cli.UimaAsServiceMonitor; import org.apache.uima.ducc.common.IServiceStatistics; import org.apache.uima.ducc.common.TcpStreamHandler; +import org.apache.uima.ducc.common.persistence.services.IStateServices; import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccProperties; import org.apache.uima.ducc.common.utils.id.DuccId; @@ -54,7 +52,6 @@ import org.apache.uima.ducc.transport.ev import org.apache.uima.ducc.transport.event.sm.ServiceDescription; import org.apache.uima.util.Level; - /** * Represents the collection of process, jobs, and such that implement a given service. */ @@ -68,7 +65,8 @@ public class ServiceSet private DuccLogger logger = DuccLogger.getLogger(this.getClass().getName(), COMPONENT_NAME); private ServiceHandler handler; - + private IStateServices stateHandler; + // key is unique id of descriptor. The descriptor inherites key from a Job's DuccId, or from // a unique-to SM key for implicit references. @@ -84,8 +82,8 @@ public class ServiceSet // For a registered service, here is my registered id DuccId id; HashMap<Long, DuccId> friendly_ids = new HashMap<Long, DuccId>(); - String history_key = "work-instances"; - String implementors_key = "implementors"; + String history_key = IStateServices.SvcProps.work_instances.pname(); + String implementors_key = IStateServices.SvcProps.implementors.pname(); // incoming nodes, for dup checking List<ServiceSet> predecessors = new ArrayList<ServiceSet>(); @@ -138,17 +136,8 @@ public class ServiceSet // registered services state files private DuccProperties job_props = null; - String props_filename = null; - String props_filename_temp = null; - File props_file; - File props_file_temp; - private DuccProperties meta_props = null; - String meta_filename = null; - String meta_filename_temp = null; - File meta_file; - File meta_file_temp; boolean deregistered = false; ServiceType service_type = ServiceType.Undefined; @@ -173,41 +162,36 @@ public class ServiceSet boolean inShutdown = false; String[] coOwners = null; + + static final String archive_key = "is_archived"; + static final String archive_flag = "true"; + // // Constructor for a registered service // - public ServiceSet(ServiceHandler handler, DuccId id, String props_filename, String meta_filename, DuccProperties props, DuccProperties meta) + public ServiceSet(ServiceHandler handler, IStateServices stateHandler, DuccId id, DuccProperties props, DuccProperties meta) { this.handler = handler; + this.stateHandler = stateHandler; this.job_props = props; this.meta_props = meta; this.id = id; - this.props_filename = props_filename; - this.props_filename_temp = props_filename + ".tmp"; - this.props_file = new File(props_filename); - this.props_file_temp = new File(props_filename_temp); - - this.meta_filename = meta_filename; - this.meta_filename_temp = meta_filename + ".tmp"; - this.meta_file = new File(meta_filename); - this.meta_file_temp = new File(meta_filename_temp); - this.service_state = ServiceState.Stopped; this.linger_time = props.getLongProperty(UiOption.ServiceLinger.pname(), linger_time); - this.key = meta.getProperty("endpoint"); + this.key = meta.getProperty(IStateServices.SvcProps.endpoint.pname()); parseEndpoint(key); - this.user = meta.getProperty("user"); - this.instances = meta.getIntProperty("instances", 1); + this.user = meta.getProperty(IStateServices.SvcProps.user.pname()); + this.instances = meta.getIntProperty(IStateServices.SvcProps.instances.pname(), 1); this.registered_instances = this.instances; - this.autostart = meta.getBooleanProperty("autostart", false); - this.ping_only = meta.getBooleanProperty("ping-only", false); - this.enabled = meta.getBooleanProperty("enabled", enabled); + this.autostart = meta.getBooleanProperty(IStateServices.SvcProps.autostart.pname(), false); + this.ping_only = meta.getBooleanProperty(IStateServices.SvcProps.ping_only.pname(), false); + this.enabled = meta.getBooleanProperty(IStateServices.SvcProps.enabled.pname(), enabled); this.service_class = ServiceClass.Registered; - this.init_failure_max = props.getIntProperty("instance_init_failures_limit", init_failure_max); - this.reference_start = meta.getBooleanProperty("reference", this.reference_start); + this.init_failure_max = props.getIntProperty(IStateServices.SvcProps.instance_init_failures_limit.pname(), init_failure_max); + this.reference_start = meta.getBooleanProperty(IStateServices.SvcProps.reference.pname(), this.reference_start); @@ -224,28 +208,28 @@ public class ServiceSet parseIndependentServices(); - meta_props.remove("references"); // Will get refreshred in upcoming OR state messages - meta_props.remove("stopped"); // obsolete flag, clean out of older registrations + meta_props.remove(IStateServices.SvcProps.references.pname()); // Will get refreshred in upcoming OR state messages + meta_props.remove(IStateServices.SvcProps.stopped.pname()); // obsolete flag, clean out of older registrations - meta_props.put("service-class", ""+service_class.decode()); - meta_props.put("service-type", ""+service_type.decode()); - meta_props.put("enabled", "" + enabled); // may not have been there in the first place - meta_props.put("service-state", ""+getState()); - meta_props.put("ping-active", "false"); - meta_props.put("service-alive", "false"); - meta_props.put("service-healthy", "false"); - meta_props.put("service-statistics", "N/A"); + meta_props.put(IStateServices.SvcProps.service_class.pname(), ""+service_class.decode()); + meta_props.put(IStateServices.SvcProps.service_type.pname(), ""+service_type.decode()); + meta_props.put(IStateServices.SvcProps.enabled.pname(), "" + enabled); // may not have been there in the first place + meta_props.put(IStateServices.SvcProps.service_state.pname(), ""+getState()); + meta_props.put(IStateServices.SvcProps.ping_active.pname(), "false"); + meta_props.put(IStateServices.SvcProps.service_alive.pname(), "false"); + meta_props.put(IStateServices.SvcProps.service_healthy.pname(), "false"); + meta_props.put(IStateServices.SvcProps.service_statistics.pname(), "N/A"); setReferenced(this.reference_start); - setLastUse(meta_props.getLongProperty("last-use", 0L)); - setLastPing(meta_props.getLongProperty("last-ping", 0L)); - setLastRunnable(meta_props.getLongProperty("last-runnable", 0L)); + setLastUse(meta_props.getLongProperty(IStateServices.SvcProps.last_use.pname(), 0L)); + setLastPing(meta_props.getLongProperty(IStateServices.SvcProps.last_ping.pname(), 0L)); + setLastRunnable(meta_props.getLongProperty(IStateServices.SvcProps.last_runnable.pname(), 0L)); if ( (!job_props.containsKey(UiOption.ProcessExecutable.pname())) && (service_type != ServiceType.UimaAs) ) { - meta_props.put("ping-only", "true"); + meta_props.put(IStateServices.SvcProps.ping_only.pname(), "true"); this.ping_only = true; } else { - meta_props.put("ping-only", "false"); + meta_props.put(IStateServices.SvcProps.ping_only.pname(), "false"); this.ping_only = false; } @@ -317,24 +301,6 @@ public class ServiceSet } - synchronized void deleteProperties() - { - String methodName = "deleteProperties"; - - // be sure to move any services that seem not to have croaked yet to history - String history = meta_props.getStringProperty(history_key, ""); - for ( Long id : friendly_ids.keySet() ) { - history = history + " " + id.toString(); - } - meta_props.put(history_key, history); - meta_props.remove(implementors_key); - - logger.info(methodName, id, "Deleting properties for unregistered service."); - ServiceManagerComponent.deleteProperties(id.toString(), meta_filename, meta_props, props_filename, job_props); - meta_filename = null; - props_filename = null; - } - synchronized Long[] getImplementors() { return implementors.keySet().toArray(new Long[implementors.size()]); @@ -489,8 +455,9 @@ public class ServiceSet * Second phase, update history, and physical metaprops. */ void bootComplete() + throws Exception { - String methodName = "bootComplete"; + //String methodName = "bootComplete"; // // During boot, inactive implementors are removed. Here we cull the implementors list to // remove stuff that didn't come in. @@ -505,39 +472,8 @@ public class ServiceSet // // must update history against stuff we used to have and don't any more // - String old_impls = meta_props.getProperty(implementors_key); - logger.info(methodName, id, "Old implementors :", old_impls); - if ( old_impls != null ) { - Map<String, String> ip = new HashMap<String, String>(); - String[] keys = old_impls.split("\\s+"); - for ( String k : keys ) ip.put(k, k); - - String history = meta_props.getProperty(history_key); - Map<String, String> hp = new HashMap<String, String>(); - if ( history != null ) { - keys = history.split("\\s+"); - for ( String k : keys ) hp.put(k, k); - } - - // here, bop through the things we used to know about, and if - // it's missing from what checked in, it's history. - for ( String k : ip.keySet() ) { - Long iid = Long.parseLong(k); - if ( ! implementors.containsKey(iid) ) { - hp.put(k, k); - } - } - - // now put the history string back into the meta props - if ( hp.size() > 0 ) { - StringBuffer sb = new StringBuffer(); - for (String s : hp.keySet() ) { - sb.append(s); - sb.append(" "); - } - meta_props.setProperty(history_key, sb.toString().trim()); - } - } + // TODO: update the history record in the meta + // // UIMA-4258 restore instance ID if this is a hot restart if ( pending_instances.size() != 0 ) { @@ -562,7 +498,7 @@ public class ServiceSet if ( isReferencedStart() && (countImplementors() == 0 ) ) { this.reference_start = false; } - saveMetaProperties(); + updateMetaProperties(); } @@ -631,11 +567,11 @@ public class ServiceSet synchronized void setLastUse(long lu) { this.last_use = lu; - meta_props.put("last-use", Long.toString(lu)); + meta_props.put(IStateServices.SvcProps.last_use.pname(), Long.toString(lu)); if ( last_use == 0 ) { - meta_props.put("last-use-readable", "Unknown"); + meta_props.put(IStateServices.SvcProps.last_use_readable.pname(), "Unknown"); } else { - meta_props.put("last-use-readable", (new Date(lu)).toString()); + meta_props.put(IStateServices.SvcProps.last_use_readable.pname(), (new Date(lu)).toString()); } } @@ -643,11 +579,11 @@ public class ServiceSet synchronized void setLastPing(long lp) { this.last_ping = lp; - meta_props.put("last-ping", Long.toString(lp)); + meta_props.put(IStateServices.SvcProps.last_ping.pname(), Long.toString(lp)); if ( last_ping == 0 ) { - meta_props.put("last-ping-readable", "Unknown"); + meta_props.put(IStateServices.SvcProps.last_ping_readable.pname(), "Unknown"); } else { - meta_props.put("last-ping-readable", (new Date(lp)).toString()); + meta_props.put(IStateServices.SvcProps.last_ping_readable.pname(), (new Date(lp)).toString()); } } @@ -655,11 +591,11 @@ public class ServiceSet synchronized void setLastRunnable(long lr) { this.last_runnable = lr; - meta_props.put("last-runnable", Long.toString(lr)); + meta_props.put(IStateServices.SvcProps.last_runnable.pname(), Long.toString(lr)); if ( last_runnable == 0 ) { - meta_props.put("last-runnable-readable", "Unknown"); + meta_props.put(IStateServices.SvcProps.last_runnable_readable.pname(), "Unknown"); } else { - meta_props.put("last-runnable-readable", (new Date(lr)).toString()); + meta_props.put(IStateServices.SvcProps.last_runnable_readable.pname(), (new Date(lr)).toString()); } } @@ -668,13 +604,13 @@ public class ServiceSet run_failures = 0; ping_failures = 0; init_failures = 0; - meta_props.remove("submit-error"); + meta_props.remove(IStateServices.SvcProps.submit_error.pname()); excessiveRunFailures = false; } synchronized void setAutostart(boolean auto) { - meta_props.setProperty("autostart", auto ? "true" : "false"); + meta_props.setProperty(IStateServices.SvcProps.autostart.pname(), auto ? "true" : "false"); this.autostart = auto; if ( auto ) { // turning this on gives benefit of the doubt on failure management @@ -760,13 +696,13 @@ public class ServiceSet synchronized void disable(String reason) { - meta_props.put("disable-reason", reason); + meta_props.put(IStateServices.SvcProps.disable_reason.pname(), reason); this.enabled = false; } synchronized void enable() { - meta_props.remove("disable-reason"); + meta_props.remove(IStateServices.SvcProps.disable_reason.pname()); resetRuntimeErrors(); this.enabled = true; } @@ -778,7 +714,7 @@ public class ServiceSet synchronized String getDisableReason() { - return meta_props.getStringProperty("disable-reason", "Unknown"); + return meta_props.getStringProperty(IStateServices.SvcProps.disable_reason.pname(), "Unknown"); } /** @@ -817,16 +753,6 @@ public class ServiceSet deregistered = true; } - String getMetaFilename() - { - return meta_filename; - } - - String getPropsFilename() - { - return props_filename; - } - // /** // * Returns the number of currently running instances // */ @@ -842,73 +768,83 @@ public class ServiceSet { return registered_instances; } - - private boolean writeProperties(DuccProperties props, File pfile, File pfile_tmp, String type) + + /** + * Service is unregistered, remove props from main DB, write them into the history DB. + */ + synchronized void deleteProperties() + throws Exception { - - String methodName = "saveProperties"; - FileOutputStream fos = null; + String methodName = "deleteProperties"; + + String ak = meta_props.getProperty(archive_key); + if ( (ak != null ) && ak.equals(archive_flag) ) { // (Migth not be set, that's ok) + // Because of races and stuff we can get called more than once but should only + // archive once. + logger.info(methodName, id, "Bypassing move to history; already moved."); + return; + } + + String history = meta_props.getStringProperty(history_key, ""); + for ( Long id : friendly_ids.keySet() ) { + history = history + " " + id.toString(); + } + meta_props.put(history_key, history); + meta_props.put(archive_key, archive_flag); - long original_size = pfile.length(); try { - if ( (!pfile.exists()) || pfile.renameTo(pfile_tmp) ) { - fos = new FileOutputStream(pfile); - props.store(fos, type + " Descriptor"); - } else { - logger.warn(methodName, id, "Cannot save", type, "properties, rename of", pfile, "to", pfile_tmp, "fails."); - if ( (!pfile.exists()) && pfile_tmp.exists() ) { - if ( !pfile_tmp.renameTo(pfile) ) { - logger.error(methodName, id, "Cannot restore", pfile_tmp, "to", pfile, "after failed update."); - } - } - } - } catch (FileNotFoundException e) { - logger.warn(methodName, id, "Cannot save", type, "properties, file does not exist."); - } catch (IOException e) { - logger.warn(methodName, id, "I/O Error saving", type, "service properties:", e); - } catch (Throwable t) { - logger.warn(methodName, id, "Unexpected Error saving", type, "service properties:", t); - } finally { - try { - if ( fos != null ) fos.close(); - long updated_size = pfile.length(); - long tmp_size = pfile_tmp.length(); - - logger.info(methodName, id, "-----> original size", original_size, "updated size", updated_size, "tmp size", tmp_size, "<-----"); - // updated size must be > 0 and tmp_size must match original size - if ( (updated_size > 0) && (original_size == tmp_size) ) { - pfile_tmp.delete(); - } else { - logger.error(methodName, id, "Update of", pfile.toString(), "failed. Original size:", original_size, "updated size", updated_size, "temp file size", tmp_size); - logger.error(methodName, id, "The updated size must be > 0 and the temp size must match the original size for sucess."); - logger.error(methodName, id, "Attempting to restore", pfile.toString(), "from", pfile_tmp.toString()); - if ( !pfile.exists() && pfile_tmp.exists() ) { - pfile_tmp.renameTo(pfile); - } - return false; - } - } catch (Throwable t) { - logger.error(methodName, id, "Cannot close", type, "properties:", t); - return false; - } + stateHandler.moveToHistory(id, job_props, meta_props); + } catch ( Exception e ) { + logger.error(methodName, id, "Could not move properties files to history: ", e); } - return true; } - private void saveProperties(DuccProperties props, File pfile, File pfile_tmp, String type) + + /** + * Save both properties in a single transaction. + */ + synchronized void storeProperties(boolean isRecovered) + throws Exception { - - String methodName = "saveProperties"; - int max = 5; - for ( int i = 0; i < max; i++ ) { - if ( writeProperties(props, pfile, pfile_tmp, type) ) return; + //String methodName = "storeProperties"; + + // no, don't store if it gets deregistered this fast + if ( isDeregistered() ) return; + + //Long strid = id.getFriendly(); + prepareMetaProperties(); // these always need houssecleaning before storing or syncing + + if ( ! isRecovered ) { // if not recovery, no need to mess with the record + stateHandler.storeProperties(id, job_props, meta_props); + } else { + stateHandler.updateJobProperties(id, (Properties) job_props); + stateHandler.updateMetaProperties(id, meta_props); } - logger.error(methodName, id, "Cannot write", pfile, "after", max, "tries. The service may not be viable after restart or in web server status."); } - synchronized void saveMetaProperties() + synchronized void updateSvcProperties() + throws Exception { - String methodName = "saveMetaProperties"; + // no more changes + if ( isDeregistered() ) return; + + stateHandler.updateJobProperties(id, (Properties) job_props); + } + + synchronized void updateMetaProperties() + throws Exception + { + // String methodName = "saveMetaProperties"; + // UIMA-4587 Why bypass, as state can still dribble in. + // if ( isDeregistered() ) return; + + prepareMetaProperties(); + stateHandler.updateMetaProperties(id, meta_props); + } + + void prepareMetaProperties() + { + // String methodName = "saveMetaProperties"; // try { // throw new IllegalStateException("Saving meta properties"); @@ -916,16 +852,9 @@ public class ServiceSet // t.printStackTrace(); // } - // UIMA-4587 why bypass? State can dribble in for a while, let's update it + // UIMA-4587 Why bypass, as state can still dribble in. // if ( isDeregistered() ) return; - if ( meta_filename == null ) { - // if this is null it was deleted and this is some kind of lingering thread updating, that - // we don't really want any more - logger.error(methodName, id, "Meta properties is deleted, bypassing attempt to save."); - return; - } - if ( implementors.size() == 0 ) { meta_props.remove(implementors_key); } else { @@ -943,44 +872,37 @@ public class ServiceSet } - meta_props.put("reference", isReferencedStart() ? "true" : "false"); - meta_props.put("autostart", isAutostart() ? "true" : "false"); + meta_props.put(IStateServices.SvcProps.reference.pname(), isReferencedStart() ? "true" : "false"); + meta_props.put(IStateServices.SvcProps.autostart.pname(), isAutostart() ? "true" : "false"); - meta_props.put("enabled", ""+enabled); - meta_props.put("service-state", ""+ getState()); - meta_props.put("ping-active", "" + (serviceMeta != null)); - meta_props.put("service-alive", "false"); - meta_props.put("service-healthy", "false"); + meta_props.put(IStateServices.SvcProps.enabled.pname(), ""+enabled); + meta_props.put(IStateServices.SvcProps.service_state.pname(), ""+ getState()); + meta_props.put(IStateServices.SvcProps.ping_active.pname(), "" + (serviceMeta != null)); + meta_props.put(IStateServices.SvcProps.service_alive.pname(), "false"); + meta_props.put(IStateServices.SvcProps.service_healthy.pname(), "false"); if ( excessiveFailures() ) { - meta_props.put("submit-error", "Service stopped by exessive failures. Initialization failures[" + init_failures + "], Runtime failures[" + run_failures + "]"); + meta_props.put(IStateServices.SvcProps.submit_error.pname(), "Service stopped by exessive failures. Initialization failures[" + init_failures + "], Runtime failures[" + run_failures + "]"); } else { - meta_props.put("service-statistics", "N/A"); + meta_props.put(IStateServices.SvcProps.service_statistics.pname(), "N/A"); } if ( serviceMeta != null ) { IServiceStatistics ss = serviceMeta.getServiceStatistics(); if ( ss != null ) { - meta_props.put("service-alive", "" + ss.isAlive()); - meta_props.put("service-healthy", "" + ss.isHealthy()); - meta_props.put("service-statistics", "" + ss.getInfo()); + meta_props.put(IStateServices.SvcProps.service_alive.pname(), "" + ss.isAlive()); + meta_props.put(IStateServices.SvcProps.service_healthy.pname(), "" + ss.isHealthy()); + meta_props.put(IStateServices.SvcProps.service_statistics.pname(), "" + ss.getInfo()); if ( ss.isAlive() ) { // UIMA-4309 setLastPing(System.currentTimeMillis()); } } } - - saveProperties(meta_props, meta_file, meta_file_temp, "Meta"); - + return; } - void saveServiceProperties() - { - saveProperties(job_props, props_file, props_file_temp, "Service"); - } - synchronized void updateInstance(long iid, long share_id, String host) { String methodName = "updateInstance"; @@ -994,7 +916,7 @@ public class ServiceSet synchronized void updateRegisteredInstances(int n) { - meta_props.setProperty("instances", Integer.toString(n)); + meta_props.setProperty(IStateServices.SvcProps.instances.pname(), Integer.toString(n)); registered_instances = n; } @@ -1053,8 +975,10 @@ public class ServiceSet synchronized void persistReferences() { + String methodName = "persistReferences"; + if ( references.size() == 0 ) { - meta_props.remove("references"); + meta_props.remove(IStateServices.SvcProps.references.pname()); } else { StringBuffer sb = new StringBuffer(); for ( DuccId id : references.keySet() ) { @@ -1062,9 +986,13 @@ public class ServiceSet sb.append(" "); } String s = sb.toString().trim(); - meta_props.setProperty("references", s); + meta_props.setProperty(IStateServices.SvcProps.references.pname(), s); + } + try { + updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, id, "Cannot update meta properties:", e); } - saveMetaProperties(); } void clearQueue() @@ -1082,7 +1010,11 @@ public class ServiceSet } handler.removeService(this); - deleteProperties(); + try { + deleteProperties(); + } catch (Exception e1) { + logger.error(methodName, id, "Cannot complete deferred delete of properties:", e1); + } if ( service_type != ServiceType.UimaAs ) { logger.info(methodName, id, "Deleting unregistered service; not clearing queue because this is not a UIMA-AS service:", key); @@ -1145,20 +1077,21 @@ public class ServiceSet } public void setErrorString(String s) + throws Exception { - meta_props.put("submit-error", s); - saveMetaProperties(); + meta_props.put(IStateServices.SvcProps.submit_error.pname(), s); + updateMetaProperties(); } public String getErrorString() { - return meta_props.getProperty("submit-error"); + return meta_props.getProperty(IStateServices.SvcProps.submit_error.pname()); } void setReferenced(boolean r) { this.reference_start = r; - meta_props.put("reference", Boolean.toString(this.reference_start)); + meta_props.put(IStateServices.SvcProps.reference.pname(), Boolean.toString(this.reference_start)); } public synchronized void reference(DuccId id) @@ -1268,8 +1201,12 @@ public class ServiceSet instances = Math.max(0, instances - 1); // prevent autostart and error handling from restarting things stop(deletions[i]); } - - saveMetaProperties(); + + try { + updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, id, "Cannot update meta properties:", e); + } } /** @@ -1304,8 +1241,8 @@ public class ServiceSet */ boolean needNextStart(JobState old, JobState current) { - String methodName = "needNextStart"; // UIMA-4587 + String methodName="needNextStart"; if ( isDeregistered() ) { logger.info(methodName, id, "Bypassing instance start because service is unregistered."); return false; @@ -1442,7 +1379,11 @@ public class ServiceSet } } - if ( save_meta ) saveMetaProperties(); + try { + if ( save_meta ) updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, id, "Error updating meta properties:", e); + } inst.setState(state); signal(inst); } @@ -1524,7 +1465,12 @@ public class ServiceSet this.service_state = new_state; if ( prev != new_state ) { logger.info(methodName, id, "State update from[" + prev + "] to[" + new_state + "] via[" + cumulative + "] Inst[" + tail + "]" ); - saveMetaProperties(); + try { + updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, id, "Error updating meta properties:", e); + } + } // Execute actions that must always occur based on the new state @@ -1862,7 +1808,7 @@ public class ServiceSet if ( isPingOnly() && (ping_failures > ping_failure_max) ) { logger.warn(methodName, id, "Stopping ping-only service due to excessive falutes:", ping_failure_max); - meta_props.put("submit-error", "Stopping ping-only service due to excessive falutes: " + ping_failure_max); + meta_props.put(IStateServices.SvcProps.submit_error.pname(), "Stopping ping-only service due to excessive falutes: " + ping_failure_max); stop(-1L); // must be -lL Long to get the right overload implementors.remove(-1L); @@ -1889,9 +1835,10 @@ public class ServiceSet serviceMeta = null; } - if ( ! inShutdown ) { - saveMetaProperties(); // no i/o during shutdown, it has to be fast and clean - // things will be cleaned up and resynced on restart + try { + updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, id, "Error updating meta properties:", e); } } @@ -2033,6 +1980,7 @@ public class ServiceSet return; } + if ( countImplementors() >= instances ) { return; } @@ -2063,7 +2011,7 @@ public class ServiceSet si.setInstanceId(find_next_instance()); long inst_ducc_id = -1L; logger.info(methodName, id, "Starting instance. Current count", countImplementors(), "needed", instances); - if ( (inst_ducc_id = si.start(props_filename, meta_props)) >= 0 ) { + if ( (inst_ducc_id = si.start(job_props, meta_props)) >= 0 ) { implementors.put(inst_ducc_id, si); handler.addInstance(this, si); signal(si); @@ -2075,7 +2023,11 @@ public class ServiceSet } } - saveMetaProperties(); + try { + updateMetaProperties(); + } catch ( Exception e ) { + logger.warn(methodName, id, "Error updating meta properties:", e); + } } /** @@ -2208,13 +2160,13 @@ public class ServiceSet sd.setLinger(linger_time); sd.setId(id.getFriendly()); sd.setUser(user); - sd.setDisableReason(meta_props.getStringProperty("disable-reason", null)); + sd.setDisableReason(meta_props.getStringProperty(IStateServices.SvcProps.disable_reason.pname(), null)); sd.setLastUse(last_use); sd.setLastPing(last_ping); // UIMA-4309 sd.setLastRunnable(last_runnable); // UIMA-4309 - sd.setRegistrationDate(meta_props.getStringProperty("registration-date", "")); + sd.setRegistrationDate(meta_props.getStringProperty(IStateServices.SvcProps.registration_date.pname(), "")); sd.setReferenceStart(reference_start); - sd.setErrorString(meta_props.getStringProperty("submit-error", null)); + sd.setErrorString(meta_props.getStringProperty(IStateServices.SvcProps.submit_error.pname(), null)); if ( serviceMeta != null ) { sd.setQueueStatistics(serviceMeta.getServiceStatistics()); @@ -2232,20 +2184,6 @@ public class ServiceSet return endpoint; } - class Starter - implements Runnable - { - ServiceInstance si; - Starter(ServiceInstance si) - { - this.si = si; - } - - public void run() { - si.start(props_filename, meta_props); - } - } - class Stopper implements Runnable { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml Tue Sep 15 14:31:24 2015 @@ -76,6 +76,41 @@ <scope>compile</scope> </dependency> + <dependency> + <groupId>com.orientechnologies</groupId> + <artifactId>orientdb-graphdb</artifactId> + <version>${orientdb.version}</version> + </dependency> + + <dependency> + <groupId>com.tinkerpop.blueprints</groupId> + <artifactId>blueprints-core</artifactId> + <version>2.6.0</version> + </dependency> + + <dependency> + <groupId>com.googlecode.concurrentlinkedhashmap</groupId> + <artifactId>concurrentlinkedhashmap-lru</artifactId> + <version>1.4.2</version> + </dependency> + + <dependency> + <groupId>com.tinkerpop.gremlin</groupId> + <artifactId>gremlin-groovy</artifactId> + <version>2.6.0</version> + </dependency> + + <dependency> + <groupId>com.orientechnologies</groupId> + <artifactId>orientdb-lucene</artifactId> + <version>${orientdb.version}</version> + </dependency> + + <dependency> + <groupId>org.webjars</groupId> + <artifactId>orientdb-studio</artifactId> + <version>${orientdb.studio.version}</version> + </dependency> Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccPerWorkItemStatistics.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccPerWorkItemStatistics.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccPerWorkItemStatistics.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccPerWorkItemStatistics.java Tue Sep 15 14:31:24 2015 @@ -37,7 +37,11 @@ public class DuccPerWorkItemStatistics i this.stddev = stddev; } - + public void setStandardDeviation(double s) + { + this.stddev = s; + } + public double getMax() { return max; } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkReservation.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkReservation.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkReservation.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkReservation.java Tue Sep 15 14:31:24 2015 @@ -35,7 +35,7 @@ public class DuccWorkReservation extends */ private static final long serialVersionUID = 1L; - private static final DuccLogger logger = DuccService.getDuccLogger(DuccWorkReservation.class.getName()); + private transient DuccLogger logger = null; private IDuccReservationMap duccReservationMap = new DuccReservationMap(); private IRationale completionRationale = null; private boolean waitForAssignment = false; @@ -50,8 +50,18 @@ public class DuccWorkReservation extends public DuccWorkReservation(DuccId duccId) { init(duccId); } + + public void initLogger() + { + // xstream won't call constructors, and the logger needs to be transient, so there's no other way + // to get it initialized. gson does call the default constructor, correctly. + // So when DuccWorkReservation is loaded from a checkpoint file by xstream this needs to be called. + if (DuccService.getDuccLogger() != null ) logger = DuccService.getDuccLogger(DuccWorkReservation.class.getName()); + } + private void init(DuccId duccId) { + initLogger(); setDuccType(DuccType.Reservation); setDuccId(duccId); setStateObject(IDuccState.ReservationState.Undefined);
