Author: degenaro Date: Tue May 20 15:09:53 2014 New Revision: 1596283 URL: http://svn.apache.org/r1596283 Log: UIMA-3825 DUCC Web Server (WS) needs to account for unusable memory
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java Tue May 20 15:09:53 2014 @@ -68,12 +68,11 @@ public class DuccWebQueryMachines extend System.out.println(" "+"aliens: "+ stringify(f.aliens)); System.out.println(" "+"swapInuse: "+ f.swapInuse); System.out.println(" "+"swapFree: "+ f.swapFree); - System.out.println(" "+"reserve: "+ f.reserve); - System.out.println(" "+"memory: "+ f.memory); + System.out.println(" "+"memoryEffective: "+ f.memoryEffective); + System.out.println(" "+"memoryTotal: "+ f.memoryTotal); System.out.println(" "+"sharesTotal: "+ f.sharesTotal); System.out.println(" "+"sharesInuse: "+ f.sharesInuse); System.out.println(" "+"ip: "+ f.ip); - System.out.println(" "+"memory: "+ f.memory); System.out.println(" "+"heartbeat: "+ f.heartbeat); } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java Tue May 20 15:09:53 2014 @@ -28,8 +28,8 @@ public class MachineFacts implements Ser public String status; public String ip; public String name; - public String reserve; - public String memory; + public String memoryEffective; + public String memoryTotal; public String swapInuse; public String swapFree; public List<String> aliens; @@ -40,8 +40,8 @@ public class MachineFacts implements Ser public MachineFacts(String status, String ip, String name, - String reserve, - String memory, + String memoryEffective, + String memoryTotal, String swapInuse, String swapFree, List<String> aliens, @@ -53,8 +53,8 @@ public class MachineFacts implements Ser this.status = status; this.ip = ip; this.name = name; - this.reserve = reserve; - this.memory = memory; + this.memoryEffective = memoryEffective; + this.memoryTotal = memoryTotal; this.swapInuse = swapInuse; this.swapFree = swapFree; this.aliens = aliens; Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java Tue May 20 15:09:53 2014 @@ -168,9 +168,9 @@ public class MachineFactsTest { if(true && m1.heartbeat.equals(m2.heartbeat) && m1.ip.equals(m2.ip) - && m1.memory.equals(m2.memory) + && m1.memoryTotal.equals(m2.memoryTotal) && m1.name.equals(m2.name) - && m1.reserve.equals(m2.reserve) + && m1.memoryEffective.equals(m2.memoryEffective) && m1.sharesInuse.equals(m2.sharesInuse) && m1.sharesTotal.equals(m2.sharesTotal) && m1.status.equals(m2.status) Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java Tue May 20 15:09:53 2014 @@ -123,7 +123,11 @@ public class DuccMachinesData { Iterator<String> iterator = nodes.iterator(); while(iterator.hasNext()) { String nodeName = (String) iterator.next(); - MachineInfo machineInfo = new MachineInfo(IDuccEnv.DUCC_NODES_FILE_PATH, "", nodeName, "", "", "", null, "", "", -1, 0); + String memTotal = ""; + String memFree = ""; + String swapInuse = ""; + String swapFree = ""; + MachineInfo machineInfo = new MachineInfo(IDuccEnv.DUCC_NODES_FILE_PATH, "", nodeName, memTotal, memFree, swapInuse, swapFree, null, "", "", -1, 0); unsortedMachines.put(machineInfo.getName(),machineInfo); } } @@ -223,21 +227,28 @@ public class DuccMachinesData { String machineName = nodeMetrics.getNodeIdentity().getName().trim(); ipToNameMap.put(ip.toString(),machineName); nameToIpMap.put(machineName,ip.toString()); - double dval = nodeMetrics.getNodeMemory().getMemTotal(); - long lval = (long) (dval/(1024*1024)+0.5); - msi.memoryTotal = lval; - msi.sharesTotal = lval/shareSize; - String memTotal = ""+lval/*+memUnits*/; - String sharesTotal = ""+lval/shareSize; - // swap: in-use - double dvalT = nodeMetrics.getNodeMemory().getSwapTotal(); - long lvalT = (long) (dvalT/(1024*1024)+0.5); - double dvalF = nodeMetrics.getNodeMemory().getSwapFree(); - long lvalF = (long) (dvalF/(1024*1024)+0.5); - - lval = lvalT - lvalF; - String swapInuse = ""+lval/*+memUnits*/; - msi.swapInuse = lval; + // mem: total + long nodeMemTotal = nodeMetrics.getNodeMemory().getMemTotal(); + logger.info(location, jobid, "node: "+machineName+" "+"memTotal: "+nodeMemTotal); + long lvalMemTotal = (long) ((1.0*nodeMemTotal)/(1024*1024)+0.5); + msi.memoryTotal = lvalMemTotal; + String memTotal = ""+lvalMemTotal/*+memUnits*/; + // mem: free + long nodeMemFree = nodeMetrics.getNodeMemory().getMemFree(); + logger.info(location, jobid, "node: "+machineName+" "+"memFree: "+nodeMemFree); + long lvalMemFree = (long) ((1.0*nodeMemFree)/(1024*1024)+0.5); + String memFree = ""+lvalMemFree/*+memUnits*/; + // shares: total + msi.sharesTotal = lvalMemFree/shareSize; + String sharesTotal = ""+lvalMemFree/shareSize; + // swap: in-usewell + double dvalSwapTotal = nodeMetrics.getNodeMemory().getSwapTotal(); + long lvalSwapTotal = (long) (dvalSwapTotal/(1024*1024)+0.5); + double dvalSwapFree = nodeMetrics.getNodeMemory().getSwapFree(); + long lvalSwapFree = (long) (dvalSwapFree/(1024*1024)+0.5); + long lvalSwapInuse = lvalSwapTotal - lvalSwapFree; + String swapInuse = ""+lvalSwapInuse/*+memUnits*/; + msi.swapInuse = lvalSwapInuse; String swapKey = ip.toString(); String swapVal = swapInuse; if(msi.swapInuse > 0) { @@ -246,10 +257,9 @@ public class DuccMachinesData { else { isSwapping.remove(swapKey); } - lval = lvalF; //String swapFree = ""+lval/*+memUnits*/; - msi.swapFree = lval; - String swapFree = ""+lval/*+memUnits*/; + msi.swapFree = lvalSwapFree; + String swapFree = ""+lvalSwapFree/*+memUnits*/; String sharesInuse = "0"; Properties shareMap = getShareMap(shareSize); try { @@ -262,7 +272,7 @@ public class DuccMachinesData { logger.warn(location, jobid, t); } List<ProcessInfo> alienPids = nodeMetrics.getRogueProcessInfoList(); - MachineInfo current = new MachineInfo("", ip.toString(), machineName, memTotal, ""+swapInuse, ""+swapFree, alienPids, sharesTotal, sharesInuse, duccEvent.getMillis(), duccEvent.getEventSize()); + MachineInfo current = new MachineInfo("", ip.toString(), machineName, memTotal, memFree, ""+swapInuse, ""+swapFree, alienPids, sharesTotal, sharesInuse, duccEvent.getMillis(), duccEvent.getEventSize()); String key = normalizeMachineName(machineName); MachineInfo previous = unsortedMachines.get(key); if(previous != null) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java Tue May 20 15:09:53 2014 @@ -42,6 +42,7 @@ public class MachineInfo implements Comp private String ip; private String name; private String memTotal; + private String memFree; private String swapInuse; private String swapFree; private List<ProcessInfo> alienPids; @@ -53,7 +54,7 @@ public class MachineInfo implements Comp private long pubSize; private long pubSizeMax; - public MachineInfo(String fileDef, String ip, String name, String memTotal, String swapInuse, String swapFree, List<ProcessInfo> alienPids, String sharesTotal, String sharesInuse, long heartbeat, long pubSize) { + public MachineInfo(String fileDef, String ip, String name, String memTotal, String memFree, String swapInuse, String swapFree, List<ProcessInfo> alienPids, String sharesTotal, String sharesInuse, long heartbeat, long pubSize) { this.fileDef = fileDef; this.ip = ip; this.name = name; @@ -124,6 +125,10 @@ public class MachineInfo implements Comp return this.memTotal; } + public String getMemFree() { + return this.memFree; + } + public String getSwapInuse() { return this.swapInuse; } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java Tue May 20 15:09:53 2014 @@ -1651,8 +1651,8 @@ public class DuccHandlerClassic extends int counter = 0; - int sumReserve = 0; - int sumMemory = 0; + int susMemoryEffective = 0; + int sumMemoryTotal = 0; int sumSwapInuse = 0; int sumSwapFree = 0; int sumAliens = 0; @@ -1672,8 +1672,8 @@ public class DuccHandlerClassic extends while(listIterator.hasNext()) { MachineFacts facts = listIterator.next(); try { - sumReserve += Integer.parseInt(facts.reserve); - sumMemory += Integer.parseInt(facts.memory); + susMemoryEffective += Integer.parseInt(facts.memoryEffective); + sumMemoryTotal += Integer.parseInt(facts.memoryTotal); sumSwapInuse += Integer.parseInt(facts.swapInuse); sumSwapFree += Integer.parseInt(facts.swapFree); sumAliens += facts.aliens.size(); @@ -1705,11 +1705,11 @@ public class DuccHandlerClassic extends row.append("</td>"); // Reserve: total row.append("<td align=\"right\">"); - row.append(""+sumReserve); + row.append(""+susMemoryEffective); row.append("</td>"); // Memory: total row.append("<td align=\"right\">"); - row.append(""+sumMemory); + row.append(""+sumMemoryTotal); row.append("</td>"); // Swap: inuse row.append("<td align=\"right\">"); @@ -1775,13 +1775,13 @@ public class DuccHandlerClassic extends row.append("<td>"); row.append(facts.name); row.append("</td>"); - // Reserve + // Memory: effective row.append("<td align=\"right\">"); - row.append(facts.reserve); + row.append(facts.memoryEffective); row.append("</td>"); // Memory: total row.append("<td align=\"right\">"); - row.append(facts.memory); + row.append(facts.memoryTotal); row.append("</td>"); // Swap: inuse sb = new StringBuffer(); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Tue May 20 15:09:53 2014 @@ -1257,8 +1257,8 @@ public class DuccHandlerJsonFormat exten JsonObject jsonResponse = new JsonObject(); JsonArray data = new JsonArray(); - int sumReserve = 0; - int sumMemory = 0; + int sumMemoryEffective = 0; + int sumMemoryTotal = 0; int sumSwapInuse = 0; int sumSwapFree = 0; int sumAliens = 0; @@ -1278,8 +1278,8 @@ public class DuccHandlerJsonFormat exten while(listIterator.hasNext()) { MachineFacts facts = listIterator.next(); try { - sumReserve += Integer.parseInt(facts.reserve); - sumMemory += Integer.parseInt(facts.memory); + sumMemoryEffective += Integer.parseInt(facts.memoryEffective); + sumMemoryTotal += Integer.parseInt(facts.memoryTotal); sumSwapInuse += Integer.parseInt(facts.swapInuse); sumSwapFree += Integer.parseInt(facts.swapFree); sumAliens += facts.aliens.size(); @@ -1300,10 +1300,10 @@ public class DuccHandlerJsonFormat exten row.add(new JsonPrimitive("")); // Name row.add(new JsonPrimitive("")); - // Reserve: total - row.add(new JsonPrimitive(sumReserve)); + // Memory: effective + row.add(new JsonPrimitive(sumMemoryEffective)); // Memory: total - row.add(new JsonPrimitive(sumMemory)); + row.add(new JsonPrimitive(sumMemoryTotal)); // Swap: inuse row.add(new JsonPrimitive(sumSwapInuse)); // Swap: free @@ -1346,10 +1346,10 @@ public class DuccHandlerJsonFormat exten row.add(new JsonPrimitive(facts.ip)); // Name row.add(new JsonPrimitive(facts.name)); - // Reserve - row.add(new JsonPrimitive(facts.reserve)); + // Memory: effective + row.add(new JsonPrimitive(facts.memoryEffective)); // Memory: total - row.add(new JsonPrimitive(facts.memory)); + row.add(new JsonPrimitive(facts.memoryTotal)); // Swap: inuse sb = new StringBuffer(); String swapping = facts.swapInuse; Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp?rev=1596283&r1=1596282&r2=1596283&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp Tue May 20 15:09:53 2014 @@ -104,8 +104,8 @@ if (table_style.equals("scroll")) { <th align="left" title="The current status">Status</th> <th align="left" title="The host IP">IP</th> <th align="left" title="The host name">Name</th> - <th align="left" title="The host reservation size (hint: specify this size to reserve entire machine), in GB" >Reserve(GB):size</th> - <th align="left" title="The host memory size, in GB" >Memory(GB):total</th> + <th align="left" title="The host effective memory size, in GB (hint: specify this size to reserve entire machine)" >Memory(GB):usable</th> + <th align="left" title="The host actual memory size, in GB" >Memory(GB):total</th> <th align="left" title="The host inuse swap size, in GB" >Swap(GB):inuse</th> <th align="left" title="The host free swap size, in GB" >Swap(GB):free</th> <th align="left" title="The number of alien PIDs (those processes not associated with a user running jobs or having a reservation)" >Alien PIDs</th> @@ -134,8 +134,8 @@ if (table_style.equals("classic")) { <th align="left" title="The current status">Status</th> <th align="left" title="The host IP">IP</th> <th align="left" title="The host name">Name</th> - <th align="left" title="The host reservation size (hint: specify this size to reserve entire machine), in GB" >Reserve(GB):size</th> - <th align="left" title="The host memory size, in GB" >Memory(GB):total</th> + <th align="left" title="The host effective memory size, in GB (hint: specify this size to reserve entire machine)" >Memory(GB):usable</th> + <th align="left" title="The host actual memory size, in GB" >Memory(GB):total</th> <th align="left" title="The host inuse swap size, in GB" >Swap(GB):inuse</th> <th align="left" title="The host free swap size, in GB" >Swap(GB):free</th> <th align="left" title="The number of alien PIDs (those processes not associated with a user running jobs or having a reservation)" >Alien PIDs</th>