Author: degenaro
Date: Wed Jun 26 12:56:39 2013
New Revision: 1496911
URL: http://svn.apache.org/r1496911
Log:
UIMA-2987 DUCC job driver (JD) should reap unprocessed work items
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DriverStatusReport.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DuccProcessWorkItemsMap.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java?rev=1496911&r1=1496910&r2=1496911&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
Wed Jun 26 12:56:39 2013
@@ -28,6 +28,7 @@ public class DuccProcessWorkItems implem
private AtomicLong done = new AtomicLong(0);
private AtomicLong error = new AtomicLong(0);
private AtomicLong retry = new AtomicLong(0);
+ private AtomicLong lost = new AtomicLong(0);
private AtomicLong preempt = new AtomicLong(0);
private AtomicLong completedMillisTotal = new AtomicLong(0);
private AtomicLong completedMillisMax = new AtomicLong(0);
@@ -77,6 +78,11 @@ public class DuccProcessWorkItems implem
undispatch();
}
+ public void lost() {
+ lost.incrementAndGet();
+ undispatch();
+ }
+
public void preempt() {
preempt.incrementAndGet();
undispatch();
@@ -133,6 +139,17 @@ public class DuccProcessWorkItems implem
}
return retVal;
}
+
+
+ public long getCountLost() {
+ long retVal = 0;
+ try {
+ retVal = lost.get();
+ }
+ catch(Throwable t) {
+ }
+ return retVal;
+ }
public long getCountPreempt() {
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java?rev=1496911&r1=1496910&r2=1496911&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcessWorkItems.java
Wed Jun 26 12:56:39 2013
@@ -26,6 +26,7 @@ public interface IDuccProcessWorkItems e
public long getCountDone();
public long getCountError();
public long getCountRetry();
+ public long getCountLost();
public long getCountPreempt();
public long getSecsAvg();
@@ -36,5 +37,6 @@ public interface IDuccProcessWorkItems e
public void done(long millis);
public void error();
public void retry();
+ public void lost();
public void preempt();
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DriverStatusReport.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DriverStatusReport.java?rev=1496911&r1=1496910&r2=1496911&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DriverStatusReport.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DriverStatusReport.java
Wed Jun 26 12:56:39 2013
@@ -401,7 +401,8 @@ public class DriverStatusReport implemen
long total = getWorkItemsTotal();
long done = getWorkItemsProcessingCompleted();
long error = getWorkItemsProcessingError();
- long todo = total - (done + error);
+ long lost = getWorkItemsLost();
+ long todo = total - (done + error + lost);
return todo;
}
@@ -539,8 +540,9 @@ public class DriverStatusReport implemen
int fetched = getWorkItemsFetched();
int completed = getWorkItemsProcessingCompleted();
int error = getWorkItemsProcessingError();
- boolean retVal = fetched != (completed + error);
- duccOut.debug(methodName, jobid, "fetched:"+fetched+"
"+"completed:"+completed+" "+"error:"+error);
+ int lost = getWorkItemsLost();
+ boolean retVal = fetched != (completed + error + lost);
+ duccOut.debug(methodName, jobid, "fetched:"+fetched+"
"+"completed:"+completed+" "+"error:"+error+" "+"lost:"+lost);
return retVal;
}
}
@@ -589,7 +591,9 @@ public class DriverStatusReport implemen
if(isTerminateDriver()) {
setDriverState(DriverState.Completed);
if(getWorkItemsProcessingError() == 0) {
-
setJobCompletion(JobCompletionType.EndOfJob, new Rationale("job driver status
reported as normal completion"));
+ if(getWorkItemsLost() == 0) {
+
setJobCompletion(JobCompletionType.EndOfJob, new Rationale("job driver status
reported as normal completion"));
+ }
}
}
break;
@@ -606,6 +610,7 @@ public class DriverStatusReport implemen
+"started:"+getWorkItemsProcessingStarted()+" "
+"completed:"+getWorkItemsProcessingCompleted()+" "
+"error:"+getWorkItemsProcessingError()+" "
+
+"lost:"+getWorkItemsLost()+" "
+"queued:"+getWorkItemsQueued()+" "
+"in-progress:"+casOperatingMap.size()+" "
+"pending:"+isPending()+" "
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DuccProcessWorkItemsMap.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DuccProcessWorkItemsMap.java?rev=1496911&r1=1496910&r2=1496911&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DuccProcessWorkItemsMap.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/DuccProcessWorkItemsMap.java
Wed Jun 26 12:56:39 2013
@@ -72,6 +72,13 @@ public class DuccProcessWorkItemsMap ext
}
}
+ public void lost(DuccId processId) {
+ getTotals().lost();
+ if(processId != null) {
+ get(processId).lost();
+ }
+ }
+
public void preempt(DuccId processId) {
getTotals().preempt();
if(processId != null) {
