Author: degenaro
Date: Tue May 20 15:09:53 2014
New Revision: 1596283
URL: http://svn.apache.org/r1596283
Log:
UIMA-3825 DUCC Web Server (WS) needs to account for unusable memory
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/DuccWebQueryMachines.java
Tue May 20 15:09:53 2014
@@ -68,12 +68,11 @@ public class DuccWebQueryMachines extend
System.out.println(" "+"aliens: "+
stringify(f.aliens));
System.out.println(" "+"swapInuse: "+
f.swapInuse);
System.out.println(" "+"swapFree: "+
f.swapFree);
- System.out.println(" "+"reserve: "+ f.reserve);
- System.out.println(" "+"memory: "+ f.memory);
+ System.out.println(" "+"memoryEffective: "+
f.memoryEffective);
+ System.out.println(" "+"memoryTotal: "+
f.memoryTotal);
System.out.println(" "+"sharesTotal: "+
f.sharesTotal);
System.out.println(" "+"sharesInuse: "+
f.sharesInuse);
System.out.println(" "+"ip: "+ f.ip);
- System.out.println(" "+"memory: "+ f.memory);
System.out.println(" "+"heartbeat: "+
f.heartbeat);
}
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/ws/json/MachineFacts.java
Tue May 20 15:09:53 2014
@@ -28,8 +28,8 @@ public class MachineFacts implements Ser
public String status;
public String ip;
public String name;
- public String reserve;
- public String memory;
+ public String memoryEffective;
+ public String memoryTotal;
public String swapInuse;
public String swapFree;
public List<String> aliens;
@@ -40,8 +40,8 @@ public class MachineFacts implements Ser
public MachineFacts(String status,
String ip,
String name,
- String reserve,
- String memory,
+ String memoryEffective,
+ String memoryTotal,
String swapInuse,
String swapFree,
List<String> aliens,
@@ -53,8 +53,8 @@ public class MachineFacts implements Ser
this.status = status;
this.ip = ip;
this.name = name;
- this.reserve = reserve;
- this.memory = memory;
+ this.memoryEffective = memoryEffective;
+ this.memoryTotal = memoryTotal;
this.swapInuse = swapInuse;
this.swapFree = swapFree;
this.aliens = aliens;
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/test/java/org/apache/uima/ducc/cli/test/MachineFactsTest.java
Tue May 20 15:09:53 2014
@@ -168,9 +168,9 @@ public class MachineFactsTest {
if(true
&& m1.heartbeat.equals(m2.heartbeat)
&& m1.ip.equals(m2.ip)
- && m1.memory.equals(m2.memory)
+ && m1.memoryTotal.equals(m2.memoryTotal)
&& m1.name.equals(m2.name)
- && m1.reserve.equals(m2.reserve)
+ && m1.memoryEffective.equals(m2.memoryEffective)
&& m1.sharesInuse.equals(m2.sharesInuse)
&& m1.sharesTotal.equals(m2.sharesTotal)
&& m1.status.equals(m2.status)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java
Tue May 20 15:09:53 2014
@@ -123,7 +123,11 @@ public class DuccMachinesData {
Iterator<String> iterator = nodes.iterator();
while(iterator.hasNext()) {
String nodeName = (String) iterator.next();
- MachineInfo machineInfo = new
MachineInfo(IDuccEnv.DUCC_NODES_FILE_PATH, "", nodeName, "", "", "", null, "",
"", -1, 0);
+ String memTotal = "";
+ String memFree = "";
+ String swapInuse = "";
+ String swapFree = "";
+ MachineInfo machineInfo = new
MachineInfo(IDuccEnv.DUCC_NODES_FILE_PATH, "", nodeName, memTotal, memFree,
swapInuse, swapFree, null, "", "", -1, 0);
unsortedMachines.put(machineInfo.getName(),machineInfo);
}
}
@@ -223,21 +227,28 @@ public class DuccMachinesData {
String machineName =
nodeMetrics.getNodeIdentity().getName().trim();
ipToNameMap.put(ip.toString(),machineName);
nameToIpMap.put(machineName,ip.toString());
- double dval = nodeMetrics.getNodeMemory().getMemTotal();
- long lval = (long) (dval/(1024*1024)+0.5);
- msi.memoryTotal = lval;
- msi.sharesTotal = lval/shareSize;
- String memTotal = ""+lval/*+memUnits*/;
- String sharesTotal = ""+lval/shareSize;
- // swap: in-use
- double dvalT = nodeMetrics.getNodeMemory().getSwapTotal();
- long lvalT = (long) (dvalT/(1024*1024)+0.5);
- double dvalF = nodeMetrics.getNodeMemory().getSwapFree();
- long lvalF = (long) (dvalF/(1024*1024)+0.5);
-
- lval = lvalT - lvalF;
- String swapInuse = ""+lval/*+memUnits*/;
- msi.swapInuse = lval;
+ // mem: total
+ long nodeMemTotal = nodeMetrics.getNodeMemory().getMemTotal();
+ logger.info(location, jobid, "node: "+machineName+"
"+"memTotal: "+nodeMemTotal);
+ long lvalMemTotal = (long) ((1.0*nodeMemTotal)/(1024*1024)+0.5);
+ msi.memoryTotal = lvalMemTotal;
+ String memTotal = ""+lvalMemTotal/*+memUnits*/;
+ // mem: free
+ long nodeMemFree = nodeMetrics.getNodeMemory().getMemFree();
+ logger.info(location, jobid, "node: "+machineName+" "+"memFree:
"+nodeMemFree);
+ long lvalMemFree = (long) ((1.0*nodeMemFree)/(1024*1024)+0.5);
+ String memFree = ""+lvalMemFree/*+memUnits*/;
+ // shares: total
+ msi.sharesTotal = lvalMemFree/shareSize;
+ String sharesTotal = ""+lvalMemFree/shareSize;
+ // swap: in-usewell
+ double dvalSwapTotal =
nodeMetrics.getNodeMemory().getSwapTotal();
+ long lvalSwapTotal = (long) (dvalSwapTotal/(1024*1024)+0.5);
+ double dvalSwapFree = nodeMetrics.getNodeMemory().getSwapFree();
+ long lvalSwapFree = (long) (dvalSwapFree/(1024*1024)+0.5);
+ long lvalSwapInuse = lvalSwapTotal - lvalSwapFree;
+ String swapInuse = ""+lvalSwapInuse/*+memUnits*/;
+ msi.swapInuse = lvalSwapInuse;
String swapKey = ip.toString();
String swapVal = swapInuse;
if(msi.swapInuse > 0) {
@@ -246,10 +257,9 @@ public class DuccMachinesData {
else {
isSwapping.remove(swapKey);
}
- lval = lvalF;
//String swapFree = ""+lval/*+memUnits*/;
- msi.swapFree = lval;
- String swapFree = ""+lval/*+memUnits*/;
+ msi.swapFree = lvalSwapFree;
+ String swapFree = ""+lvalSwapFree/*+memUnits*/;
String sharesInuse = "0";
Properties shareMap = getShareMap(shareSize);
try {
@@ -262,7 +272,7 @@ public class DuccMachinesData {
logger.warn(location, jobid, t);
}
List<ProcessInfo> alienPids =
nodeMetrics.getRogueProcessInfoList();
- MachineInfo current = new MachineInfo("", ip.toString(),
machineName, memTotal, ""+swapInuse, ""+swapFree, alienPids, sharesTotal,
sharesInuse, duccEvent.getMillis(), duccEvent.getEventSize());
+ MachineInfo current = new MachineInfo("", ip.toString(),
machineName, memTotal, memFree, ""+swapInuse, ""+swapFree, alienPids,
sharesTotal, sharesInuse, duccEvent.getMillis(), duccEvent.getEventSize());
String key = normalizeMachineName(machineName);
MachineInfo previous = unsortedMachines.get(key);
if(previous != null) {
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/MachineInfo.java
Tue May 20 15:09:53 2014
@@ -42,6 +42,7 @@ public class MachineInfo implements Comp
private String ip;
private String name;
private String memTotal;
+ private String memFree;
private String swapInuse;
private String swapFree;
private List<ProcessInfo> alienPids;
@@ -53,7 +54,7 @@ public class MachineInfo implements Comp
private long pubSize;
private long pubSizeMax;
- public MachineInfo(String fileDef, String ip, String name, String
memTotal, String swapInuse, String swapFree, List<ProcessInfo> alienPids,
String sharesTotal, String sharesInuse, long heartbeat, long pubSize) {
+ public MachineInfo(String fileDef, String ip, String name, String
memTotal, String memFree, String swapInuse, String swapFree, List<ProcessInfo>
alienPids, String sharesTotal, String sharesInuse, long heartbeat, long
pubSize) {
this.fileDef = fileDef;
this.ip = ip;
this.name = name;
@@ -124,6 +125,10 @@ public class MachineInfo implements Comp
return this.memTotal;
}
+ public String getMemFree() {
+ return this.memFree;
+ }
+
public String getSwapInuse() {
return this.swapInuse;
}
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
Tue May 20 15:09:53 2014
@@ -1651,8 +1651,8 @@ public class DuccHandlerClassic extends
int counter = 0;
- int sumReserve = 0;
- int sumMemory = 0;
+ int susMemoryEffective = 0;
+ int sumMemoryTotal = 0;
int sumSwapInuse = 0;
int sumSwapFree = 0;
int sumAliens = 0;
@@ -1672,8 +1672,8 @@ public class DuccHandlerClassic extends
while(listIterator.hasNext()) {
MachineFacts facts = listIterator.next();
try {
- sumReserve +=
Integer.parseInt(facts.reserve);
- sumMemory +=
Integer.parseInt(facts.memory);
+ susMemoryEffective +=
Integer.parseInt(facts.memoryEffective);
+ sumMemoryTotal +=
Integer.parseInt(facts.memoryTotal);
sumSwapInuse +=
Integer.parseInt(facts.swapInuse);
sumSwapFree +=
Integer.parseInt(facts.swapFree);
sumAliens += facts.aliens.size();
@@ -1705,11 +1705,11 @@ public class DuccHandlerClassic extends
row.append("</td>");
// Reserve: total
row.append("<td align=\"right\">");
- row.append(""+sumReserve);
+ row.append(""+susMemoryEffective);
row.append("</td>");
// Memory: total
row.append("<td align=\"right\">");
- row.append(""+sumMemory);
+ row.append(""+sumMemoryTotal);
row.append("</td>");
// Swap: inuse
row.append("<td align=\"right\">");
@@ -1775,13 +1775,13 @@ public class DuccHandlerClassic extends
row.append("<td>");
row.append(facts.name);
row.append("</td>");
- // Reserve
+ // Memory: effective
row.append("<td align=\"right\">");
- row.append(facts.reserve);
+ row.append(facts.memoryEffective);
row.append("</td>");
// Memory: total
row.append("<td align=\"right\">");
- row.append(facts.memory);
+ row.append(facts.memoryTotal);
row.append("</td>");
// Swap: inuse
sb = new StringBuffer();
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
Tue May 20 15:09:53 2014
@@ -1257,8 +1257,8 @@ public class DuccHandlerJsonFormat exten
JsonObject jsonResponse = new JsonObject();
JsonArray data = new JsonArray();
- int sumReserve = 0;
- int sumMemory = 0;
+ int sumMemoryEffective = 0;
+ int sumMemoryTotal = 0;
int sumSwapInuse = 0;
int sumSwapFree = 0;
int sumAliens = 0;
@@ -1278,8 +1278,8 @@ public class DuccHandlerJsonFormat exten
while(listIterator.hasNext()) {
MachineFacts facts = listIterator.next();
try {
- sumReserve +=
Integer.parseInt(facts.reserve);
- sumMemory +=
Integer.parseInt(facts.memory);
+ sumMemoryEffective +=
Integer.parseInt(facts.memoryEffective);
+ sumMemoryTotal +=
Integer.parseInt(facts.memoryTotal);
sumSwapInuse +=
Integer.parseInt(facts.swapInuse);
sumSwapFree +=
Integer.parseInt(facts.swapFree);
sumAliens += facts.aliens.size();
@@ -1300,10 +1300,10 @@ public class DuccHandlerJsonFormat exten
row.add(new JsonPrimitive(""));
// Name
row.add(new JsonPrimitive(""));
- // Reserve: total
- row.add(new JsonPrimitive(sumReserve));
+ // Memory: effective
+ row.add(new JsonPrimitive(sumMemoryEffective));
// Memory: total
- row.add(new JsonPrimitive(sumMemory));
+ row.add(new JsonPrimitive(sumMemoryTotal));
// Swap: inuse
row.add(new JsonPrimitive(sumSwapInuse));
// Swap: free
@@ -1346,10 +1346,10 @@ public class DuccHandlerJsonFormat exten
row.add(new JsonPrimitive(facts.ip));
// Name
row.add(new JsonPrimitive(facts.name));
- // Reserve
- row.add(new JsonPrimitive(facts.reserve));
+ // Memory: effective
+ row.add(new
JsonPrimitive(facts.memoryEffective));
// Memory: total
- row.add(new JsonPrimitive(facts.memory));
+ row.add(new JsonPrimitive(facts.memoryTotal));
// Swap: inuse
sb = new StringBuffer();
String swapping = facts.swapInuse;
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
URL:
http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp?rev=1596283&r1=1596282&r2=1596283&view=diff
==============================================================================
---
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
(original)
+++
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/system.machines.jsp
Tue May 20 15:09:53 2014
@@ -104,8 +104,8 @@ if (table_style.equals("scroll")) {
<th align="left" title="The current status">Status</th>
<th align="left" title="The host IP">IP</th>
<th align="left" title="The host name">Name</th>
- <th align="left" title="The host reservation size (hint: specify this
size to reserve entire machine), in GB" >Reserve(GB):size</th>
- <th align="left" title="The host memory size, in GB"
>Memory(GB):total</th>
+ <th align="left" title="The host effective memory size, in GB (hint:
specify this size to reserve entire machine)" >Memory(GB):usable</th>
+ <th align="left" title="The host actual memory size, in GB"
>Memory(GB):total</th>
<th align="left" title="The host inuse swap size, in GB"
>Swap(GB):inuse</th>
<th align="left" title="The host free swap size, in GB"
>Swap(GB):free</th>
<th align="left" title="The number of alien PIDs (those processes not
associated with a user running jobs or having a reservation)" >Alien PIDs</th>
@@ -134,8 +134,8 @@ if (table_style.equals("classic")) {
<th align="left" title="The current status">Status</th>
<th align="left" title="The host IP">IP</th>
<th align="left" title="The host name">Name</th>
- <th align="left" title="The host reservation size (hint:
specify this size to reserve entire machine), in GB" >Reserve(GB):size</th>
- <th align="left" title="The host memory size, in GB"
>Memory(GB):total</th>
+ <th align="left" title="The host effective memory size, in GB
(hint: specify this size to reserve entire machine)" >Memory(GB):usable</th>
+ <th align="left" title="The host actual memory size, in GB"
>Memory(GB):total</th>
<th align="left" title="The host inuse swap size, in GB"
>Swap(GB):inuse</th>
<th align="left" title="The host free swap size, in GB"
>Swap(GB):free</th>
<th align="left" title="The number of alien PIDs (those
processes not associated with a user running jobs or having a reservation)"
>Alien PIDs</th>
