Author: challngr Date: Wed May 29 18:05:59 2013 New Revision: 1487580 URL: http://svn.apache.org/r1487580 Log: UIMA-2593 Fix RM overcounting a dead node with work in limbo. Also fix the system tests to include the random number seed (found while working on this boog).
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.fragment Wed May 29 18:05:59 2013 @@ -3,6 +3,11 @@ # Compression of 96 and spread 900 gives a good 15-20 minute test on test system with # 136 15-G shares +random.seed = 0 # a number, for determinate randoms + # or TOD, and the seed will use + # current time of day + + src.dir = jobs.in # where the jobs are dest.dir = jobs # where to put prepared jobs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-15-min/jobs.prepare.mixed Wed May 29 18:05:59 2013 @@ -3,6 +3,11 @@ # Compression of 96 and spread 900 gives a good 15-20 minute test on test system with # 136 15-G shares +random.seed = 0 # a number, for determinate randoms + # or TOD, and the seed will use + # current time of day + + src.dir = jobs.in # where the jobs are dest.dir = jobs # where to put prepared jobs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare Wed May 29 18:05:59 2013 @@ -1,6 +1,11 @@ # These jobs represent approximately 144 hours real time; adjust compression accordingly # Compresion of 12 produces an approximately 12 hour test on a system with 136 15G shares +random.seed = 0 # a number, for determinate randoms + # or TOD, and the seed will use + # current time of day + + src.dir = jobs.in # where the jobs are dest.dir = jobs # where to put prepared jobs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.fragment Wed May 29 18:05:59 2013 @@ -1,6 +1,11 @@ # These jobs represent approximately 144 hours real time; adjust compression accordingly # Compresion of 12 produces an approximately 12 hour test on a system with 136 15G shares +random.seed = 0 # a number, for determinate randoms + # or TOD, and the seed will use + # current time of day + + src.dir = projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs.in # where the jobs are dest.dir = projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs # where to put prepared jobs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-24-hrs/jobs.prepare.mixed Wed May 29 18:05:59 2013 @@ -1,6 +1,11 @@ # These jobs represent approximately 144 hours real time; adjust compression accordingly # Compresion of 12 and spread of 42000 produces a good 12 hour test on a system with 136 15G shares +random.seed = 0 # a number, for determinate randoms + # or TOD, and the seed will use + # current time of day + + src.dir = projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs.in # where the jobs are dest.dir = projects/ducc/ducc_test/test/blade-2010-10-13-18/jobs # where to put prepared jobs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-examples/src/main/systemtest/mega-30-min/jobs.prepare Wed May 29 18:05:59 2013 @@ -3,6 +3,11 @@ # Compression of 48 and spread 1800 gives a good 30 minute test on test system with # 136 15-G shares +random.seed = 0 # a number, for determinate randoms + # or TOD, and the seed will use + # current time of day + + src.dir = jobs.in # where the jobs are dest.dir = jobs # where to put prepared jobs Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java Wed May 29 18:05:59 2013 @@ -1219,6 +1219,8 @@ class NodePool int order = j.getShareOrder(); int given = 0; + logger.debug(methodName, j.getId(), "counted", counted, "current", current, "needed", needed, "order", order, "given", given); + if ( needed > 0 ) { whatof: { for ( int i = order; i < getArraySize(); i++ ) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1487580&r1=1487579&r2=1487580&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java Wed May 29 18:05:59 2013 @@ -980,9 +980,9 @@ public class RmJob c++; } - c = Math.max(c, assignedShares.size()); // if job is ending we could be fragmented and have to be - // careful not to underestimate, or we end up possibly - // evicting something that should be left alone. + c = Math.max(c, assignedShares.size() - pendingRemoves.size()); // if job is ending we could be fragmented and have to be + // careful not to underestimate, or we end up possibly + // evicting something that should be left alone. // // If waiting for initialization, we have to cap as well on the maximum number of shares