Author: degenaro
Date: Tue Sep 11 18:00:52 2018
New Revision: 1840596
URL: http://svn.apache.org/viewvc?rev=1840596&view=rev
Log:
UIMA-5851 DUCC Web Server (WS) displays ever increasing Processing Time on Work
Items tab
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItemsV1.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ProcessAccounting.java
Tue Sep 11 18:00:52 2018
@@ -564,107 +564,6 @@ public class ProcessAccounting {
logger.trace(methodName, job.getDuccId(),
messages.fetch("exit"));
}
- private ITimeWindow makeTimeWindow(String ts) {
- ITimeWindow tw = new TimeWindow();
- tw.setStart(ts);
- tw.setEnd(ts);
- return tw;
- }
-
- private void initStop(IDuccWorkJob job, IDuccProcess process) {
- String ts = TimeStamp.getCurrentMillis();
- ITimeWindow twi = process.getTimeWindowInit();
- if(twi == null) {
- twi = makeTimeWindow(ts);
- process.setTimeWindowInit(twi);
- }
- else {
- long i0 = twi.getStartLong();
- long i1 = twi.getEndLong();
- if(i0 != i1) {
- if(i1 < i0) {
- twi.setEnd(ts);
- }
- }
- }
- }
-
- private void runStart(IDuccWorkJob job, IDuccProcess process) {
- ITimeWindow twi = process.getTimeWindowInit();
- ITimeWindow twr = makeTimeWindow(twi.getEnd());
- process.setTimeWindowRun(twr);
- }
-
- private void runStop(IDuccWorkJob job, IDuccProcess process) {
- String ts = TimeStamp.getCurrentMillis();
- ITimeWindow twi = process.getTimeWindowInit();
- if(twi == null) {
- twi = makeTimeWindow(ts);
- process.setTimeWindowRun(twi);
- }
- ITimeWindow twr = process.getTimeWindowRun();
- if(twr == null) {
- twr = makeTimeWindow(twi.getEnd());
- process.setTimeWindowRun(twr);
- }
- else {
- long r0 = twr.getStartLong();
- long r1 = twr.getEndLong();
- if(r0 != r1) {
- if(r1 < r0) {
- twr.setEnd(ts);
- }
- }
- }
- adjustWindows(job, process);
- adjustRunTime(job, process);
- }
-
- // <uima-3351>
- private void adjustRunTime(IDuccWorkJob job, IDuccProcess process) {
- switch(job.getDuccType()) {
- case Job:
- if(!process.isAssignedWork()) {
- ITimeWindow twr = process.getTimeWindowRun();
- if(twr == null) {
- twr = new TimeWindow();
- process.setTimeWindowRun(twr);
- }
- long time = 0;
- twr.setStartLong(time);
- twr.setEndLong(time);
- }
- break;
- default:
- break;
- }
- }
- // </uima-3351>
-
- private void adjustWindows(IDuccWorkJob job, IDuccProcess process) {
- String methodName = "adjustWindows";
- ITimeWindow twi = process.getTimeWindowInit();
- long i0 = twi.getStartLong();
- long i1 = twi.getEndLong();
- ITimeWindow twr = process.getTimeWindowRun();
- long r0 = twr.getStartLong();
- long r1 = twr.getEndLong();
- if(i0 != i1) {
- if(r0 != r1) {
- if(r0 < i1) {
- logger.warn(methodName,
job.getDuccId(), process.getDuccId(), "run-start: "+r0+" -> "+i1);
- r0 = i1;
- twr.setStartLong(r0);
- if(r1 < r0) {
- logger.warn(methodName,
job.getDuccId(), process.getDuccId(), "run-end: "+r1+" -> "+r0);
- r1 = r0;
- twr.setEndLong(r1);
- }
- }
- }
- }
- }
-
private void updateProcessInitilization(IDuccWorkJob job, IDuccProcess
inventoryProcess, IDuccProcess process) {
switch(inventoryProcess.getProcessState()) {
case Started:
@@ -689,10 +588,127 @@ public class ProcessAccounting {
break;
}
}
-
+
+ private boolean isInitializing(ProcessState ps) {
+ boolean retVal = false;
+ if(ps != null) {
+ switch(ps) {
+ case Starting: // Process Manager sent
request to start the Process
+ case Started: // Process PID is available
+ case Initializing: // Process
Agent is initializing process
+ retVal = true;
+ break;
+ default:
+ break;
+ }
+ }
+ return retVal;
+ }
+
+ private boolean isRunning(ProcessState ps) {
+ boolean retVal = false;
+ if(ps != null) {
+ switch(ps) {
+ case Running: // Process
Agent is processing work items
+ retVal = true;
+ break;
+ default:
+ break;
+ }
+ }
+ return retVal;
+ }
+
+
+ private boolean isCompleted(ProcessState ps) {
+ boolean retVal = false;
+ if(ps != null) {
+ switch(ps) {
+ case LaunchFailed: // Process
Agent reports process launch failed
+ case Stopped: // Process
Agent reports process stopped
+ case Failed: // Process
Agent reports process failed
+ case FailedInitialization: // Process Agent
reports process failed initialization
+ case InitializationTimeout: // Process Agent reports
process initialization timeout
+ case Killed:
+ retVal = true;
+ break;
+ default:
+ break;
+ }
+ }
+ return retVal;
+ }
+
+ /*
+ * Agent reports initialization time, but not run time so make
adjustment accordingly
+ */
+ private void adjustProcessTime(IDuccWorkJob job, IDuccProcess
inventoryProcess, IDuccProcess process) {
+ String methodName = "adjustProcessTime";
+ logger.trace(methodName, job.getDuccId(),
messages.fetch("enter"));
+ long ts = System.currentTimeMillis();
+ DuccId jobId = job.getDuccId();
+ DuccId prcId = process.getDuccId();
+ // orchestrator process state
+ ProcessState ops = process.getProcessState();
+ if(ops == null) {
+ ops = ProcessState.Undefined;
+ process.setProcessState(ops);
+ }
+ // inventory process state
+ ProcessState ips = inventoryProcess.getProcessState();
+ if(ips == null) {
+ ips = ProcessState.Undefined;
+ inventoryProcess.setProcessState(ips);
+ }
+ ITimeWindow twi = inventoryProcess.getTimeWindowInit();
+ if(twi == null) {
+ twi = new TimeWindow();
+ inventoryProcess.setTimeWindowInit(twi);
+ }
+ long twis = twi.getStartLong();
+ long twie = twi.getEndLong();
+ ITimeWindow twr = inventoryProcess.getTimeWindowRun();
+ if(twr == null) {
+ twr = new TimeWindow();
+ inventoryProcess.setTimeWindowRun(twr);
+ }
+ long twrs = twr.getStartLong();
+ long twre = twr.getEndLong();
+ logger.debug(methodName, jobId, prcId, "raw", ips, ops, twis,
twie, twrs, twre);
+ if(isInitializing(ips)) {
+ // no adjustment
+ }
+ else if(isRunning(ips)) {
+ if(twrs < 0) {
+ twr.setStartLong(twie);
+ twrs = twr.getStartLong();
+ }
+ }
+ else if(isCompleted(ips)) {
+ if(twrs < 0) {
+ twr.setStartLong(twie);
+ twrs = twr.getStartLong();
+ }
+ if(isCompleted(ops)) {
+ ITimeWindow tw = process.getTimeWindowRun();
+ twre = tw.getEndLong();
+ twr.setEndLong(twre);
+ }
+ else {
+ if(twre < 0) {
+ twr.setEndLong(ts);
+ twre = twr.getEndLong();
+ }
+ }
+ }
+ logger.debug(methodName, jobId, prcId, "adj", ips, ops, twis,
twie, twrs, twre);
+ logger.trace(methodName, job.getDuccId(),
messages.fetch("exit"));
+ }
+
private void updateProcessTime(IDuccWorkJob job, IDuccProcess
inventoryProcess, IDuccProcess process) {
String methodName = "updateProcessTime";
logger.trace(methodName, job.getDuccId(),
messages.fetch("enter"));
+ adjustProcessTime(job,inventoryProcess,process);
switch(inventoryProcess.getProcessState()) {
case Starting: // Process Manager sent request to
start the Process
case Started: // Process PID is available
@@ -701,8 +717,6 @@ public class ProcessAccounting {
break;
case Running: // Process Agent is
processing work items
copyTimeInit(inventoryProcess, process);
- initStop(job, process);
- runStart(job, process);
copyTimeRun(inventoryProcess, process);
break;
case LaunchFailed: // Process Agent
reports process launch failed
@@ -713,8 +727,6 @@ public class ProcessAccounting {
case Killed: // Agent forcefully
killed the process
copyTimeInit(inventoryProcess, process);
copyTimeRun(inventoryProcess, process);
- initStop(job, process);
- runStop(job, process);
break;
case Undefined:
break;
@@ -779,6 +791,8 @@ public class ProcessAccounting {
}
else {
logger.trace(methodName, jobId, process.getDuccId(), "active");
+
// Process Init & Run times (first, with respect to previous process state if
any)
+
updateProcessTime(job, inventoryProcess, process);
// PID
copyInventoryPID(job, inventoryProcess, process);
// Scheduler State
@@ -789,8 +803,6 @@ public class ProcessAccounting {
copyReasonForStoppingProcess(job, inventoryProcess, process);
// Process Exit code
copyProcessExitCode(job, inventoryProcess, process);
-
// Process Init & Run times
-
updateProcessTime(job, inventoryProcess, process);
// Process Initialization State
updateProcessInitilization(job, inventoryProcess, process);
// Process Pipeline Components State
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
Tue Sep 11 18:00:52 2018
@@ -337,6 +337,18 @@ public class DuccProcess implements IDuc
}
+ public boolean isActiveWork() {
+ boolean retVal = true;
+ IDuccProcessWorkItems pwi = getProcessWorkItems();
+ if(pwi == null) {
+ retVal = false;
+ }
+ else {
+ retVal = pwi.isActiveWork();
+ }
+ return retVal;
+ }
+
public boolean isActive() {
boolean retVal = false;
switch(processState) {
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
Tue Sep 11 18:00:52 2018
@@ -50,6 +50,7 @@ public class DuccProcessWorkItems implem
doneMillisMin.set(pi.getMin());
}
+ @Override
public boolean isAssignedWork() {
boolean retVal = true;
if((getCountDispatch() == 0)
@@ -62,7 +63,17 @@ public class DuccProcessWorkItems implem
}
return retVal;
}
-
+
+ @Override
+ public boolean isActiveWork() {
+ boolean retVal = true;
+ if((getCountDispatch() == 0)
+ ) {
+ retVal = false;
+ }
+ return retVal;
+ }
+
@Override
public void setCountDispatch(long value) {
dispatch.set(value);
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItemsV1.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItemsV1.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItemsV1.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItemsV1.java
Tue Sep 11 18:00:52 2018
@@ -312,5 +312,15 @@ public class DuccProcessWorkItemsV1 impl
//V2
return 0;
}
+
+ @Override
+ public boolean isActiveWork() {
+ boolean retVal = true;
+ if((getCountDispatch() == 0)
+ ) {
+ retVal = false;
+ }
+ return retVal;
+ }
}
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
Tue Sep 11 18:00:52 2018
@@ -77,6 +77,7 @@ public interface IDuccProcess extends Se
public boolean isVolunteered();
public boolean isComplete();
+ public boolean isActiveWork();
public boolean isAssignedWork();
public boolean isDeallocated();
Modified:
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
Tue Sep 11 18:00:52 2018
@@ -22,6 +22,7 @@ import java.io.Serializable;
public interface IDuccProcessWorkItems extends Serializable {
+ public boolean isActiveWork();
public boolean isAssignedWork();
public void setCountDispatch(long value);
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Tue Sep 11 18:00:52 2018
@@ -442,6 +442,11 @@ public class DuccHandler extends DuccAbs
return Helper.getSchedulerReason(job, process);
}
+
+ private boolean isIdleJobProcess(IDuccWorkJob job, IDuccProcess
process, AllocationType type) {
+ return Helper.isIdleJobProcess(job, process, type);
+ }
+
private String getStateAgent(IDuccWorkJob job, IDuccProcess process) {
return Helper.getAgentState(job, process);
}
@@ -806,6 +811,9 @@ public class DuccHandler extends DuccAbs
index++; // jp.08
cbList[index].append("<td>");
String stateAgent = getStateAgent(job,process);
+ if(isIdleJobProcess(job,process,sType)) {
+ stateAgent = process.getProcessState()+":Idle";
+ }
cbList[index].append(stateAgent);
logAppend(index,"stateAgent",stateAgent);
cbList[index].append("</td>");
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java?rev=1840596&r1=1840595&r2=1840596&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/Helper.java
Tue Sep 11 18:00:52 2018
@@ -48,10 +48,10 @@ import org.apache.uima.ducc.ws.DuccMachi
import org.apache.uima.ducc.ws.MachineInfo;
import org.apache.uima.ducc.ws.types.NodeId;
import org.apache.uima.ducc.ws.utils.FormatHelper;
-import org.apache.uima.ducc.ws.utils.UrlHelper;
import org.apache.uima.ducc.ws.utils.FormatHelper.Precision;
import org.apache.uima.ducc.ws.utils.LinuxSignals;
import org.apache.uima.ducc.ws.utils.LinuxSignals.Signal;
+import org.apache.uima.ducc.ws.utils.UrlHelper;
import org.apache.uima.ducc.ws.utils.alien.EffectiveUser;
import org.apache.uima.ducc.ws.utils.alien.FileInfo;
import org.apache.uima.ducc.ws.utils.alien.OsProxy;
@@ -470,56 +470,30 @@ public class Helper {
public static String getTimeRun(IDuccWorkJob job, IDuccProcess process,
AllocationType type) {
String location = "getTimeRun";
StringBuffer sb = new StringBuffer();
- if(process != null) {
- String runTime = "00";
- // <UIMA-3351>
- boolean useTimeRun = true;
- switch(type) {
- case SPC:
- break;
- case SPU:
- break;
- case MR:
- break;
- case JD:
- break;
- case UIMA:
- if(!process.isAssignedWork()) {
- useTimeRun = false;
- }
- break;
- default:
- break;
- }
- // </UIMA-3351>
- if(useTimeRun) {
- try {
- TimeWindow t = (TimeWindow)
process.getTimeWindowRun();
- if(t != null) {
- long now =
System.currentTimeMillis();
- String tS = t.getStart(""+now);
- String tE = t.getEnd(""+now);
- runTime =
Helper.getDuration(jobid,tE,tS,Precision.Whole);
- }
- }
- catch(Exception e) {
- duccLogger.trace(location, jobid, "no
worries", e);
- }
- catch(Throwable t) {
- duccLogger.trace(location, jobid, "no
worries", t);
+ String runTime = "00";
+ try {
+ if(process != null) {
+ long now = System.currentTimeMillis();
+ TimeWindow t = (TimeWindow)
process.getTimeWindowRun();
+ String tS = t.getStart(""+now);
+ String tE = t.getEnd(""+now);
+ if(isTimeRunEstimated(job,process,type)) {
+ tE = ""+now;
}
+ runTime =
Helper.getDuration(jobid,tE,tS,Precision.Whole);
}
- sb.append(runTime);
}
+ catch(Exception e) {
+ duccLogger.trace(location, jobid, "no worries", e);
+ }
+ sb.append(runTime);
return sb.toString();
}
- public static boolean isTimeRunEstimated(IDuccWorkJob job, IDuccProcess
process, AllocationType type) {
- String location = "isTimeRunEstimated";
+ public static boolean isIdleJobProcess(IDuccWorkJob job, IDuccProcess
process, AllocationType type) {
+ String location = "isIdleJobProcess";
boolean retVal = false;
if(process != null) {
- // <UIMA-3351>
- boolean useTimeRun = true;
switch(type) {
case SPC:
break;
@@ -530,30 +504,29 @@ public class Helper {
case JD:
break;
case UIMA:
- if(!process.isAssignedWork()) {
- useTimeRun = false;
+ switch(process.getProcessState()) {
+ case Running:
+ boolean busy = process.isActiveWork();
+ duccLogger.debug(location,
job.getDuccId(), process.getDuccId(), process.getProcessState(), !busy);
+ if(!busy) {
+ retVal = true;
+ }
+ break;
+ default:
+ break;
}
break;
default:
break;
}
- // </UIMA-3351>
- if(useTimeRun) {
- try {
- TimeWindow t = (TimeWindow)
process.getTimeWindowRun();
- if(t != null) {
- if(t.isEstimated()) {
- retVal = true;
- }
- }
- }
- catch(Exception e) {
- duccLogger.trace(location, jobid, "no
worries", e);
- }
- catch(Throwable t) {
- duccLogger.trace(location, jobid, "no
worries", t);
- }
- }
+ }
+ return retVal;
+ }
+
+ public static boolean isTimeRunEstimated(IDuccWorkJob job, IDuccProcess
process, AllocationType type) {
+ boolean retVal = true;
+ if(process.isComplete()) {
+ retVal = false;
}
return retVal;
}
