Author: degenaro Date: Mon Feb 1 16:32:01 2016 New Revision: 1727969 URL: http://svn.apache.org/viewvc?rev=1727969&view=rev Log: UIMA-4776 DUCC Web Server (WS) should key machines by IP address not host name
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccMachinesData.java Mon Feb 1 16:32:01 2016 @@ -18,7 +18,6 @@ */ package org.apache.uima.ducc.ws; -import java.net.InetAddress; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -52,21 +51,19 @@ public class DuccMachinesData { private static DuccLogger logger = DuccLoggerComponents.getWsLogger(DuccMachinesData.class.getName()); private static DuccId jobid = null; - private static ConcurrentSkipListMap<MachineInfo,String> sortedMachines = new ConcurrentSkipListMap<MachineInfo,String>(); - private static ConcurrentSkipListMap<String,MachineInfo> unsortedMachines = new ConcurrentSkipListMap<String,MachineInfo>(); - private static ConcurrentSkipListMap<String,MachineSummaryInfo> summaryMachines = new ConcurrentSkipListMap<String,MachineSummaryInfo>(); + private static ConcurrentSkipListMap<MachineInfo,Ip> sortedMachines = new ConcurrentSkipListMap<MachineInfo,Ip>(); + private static ConcurrentSkipListMap<Ip,MachineInfo> unsortedMachines = new ConcurrentSkipListMap<Ip,MachineInfo>(); + private static ConcurrentSkipListMap<Ip,MachineSummaryInfo> summaryMachines = new ConcurrentSkipListMap<Ip,MachineSummaryInfo>(); private static AtomicLong memTotal = new AtomicLong(0); private static AtomicLong memFree = new AtomicLong(0); private static AtomicLong swapInuse = new AtomicLong(0); private static AtomicLong swapFree = new AtomicLong(0); - - private String domain = ""; private static DuccMachinesData duccMachinesData = new DuccMachinesData(); - private static ConcurrentSkipListMap<String,String> ipToNameMap = new ConcurrentSkipListMap<String,String>(); - private static ConcurrentSkipListMap<String,String> nameToIpMap = new ConcurrentSkipListMap<String,String>(); + private static ConcurrentSkipListMap<Ip,NodeId> ipToNameMap = new ConcurrentSkipListMap<Ip,NodeId>(); + private static ConcurrentSkipListMap<NodeId,Ip> nameToIpMap = new ConcurrentSkipListMap<NodeId,Ip>(); private static ConcurrentSkipListMap<String,String> isSwapping = new ConcurrentSkipListMap<String,String>(); private static ConcurrentSkipListMap<String,TreeMap<String,NodeUsersInfo>> ipToNodeUsersInfoMap = new ConcurrentSkipListMap<String,TreeMap<String,NodeUsersInfo>>(); @@ -79,12 +76,12 @@ public class DuccMachinesData { return isSwapping.containsKey(ip); } - public ConcurrentSkipListMap<String,MachineInfo> getMachines() { + public ConcurrentSkipListMap<Ip,MachineInfo> getMachines() { return unsortedMachines; } - public ConcurrentSkipListMap<MachineInfo,String> getSortedMachines() { - ConcurrentSkipListMap<MachineInfo,String> retVal = sortedMachines; + public ConcurrentSkipListMap<MachineInfo,Ip> getSortedMachines() { + ConcurrentSkipListMap<MachineInfo,Ip> retVal = sortedMachines; return retVal; } @@ -92,9 +89,9 @@ public class DuccMachinesData { String location = "updateSortedMachines"; logger.debug(location, jobid, "start"); try { - ConcurrentSkipListMap<MachineInfo,String> map = new ConcurrentSkipListMap<MachineInfo,String>(); - for(Entry<String, MachineInfo> entry : unsortedMachines.entrySet()) { - String value = entry.getKey(); + ConcurrentSkipListMap<MachineInfo,Ip> map = new ConcurrentSkipListMap<MachineInfo,Ip>(); + for(Entry<Ip,MachineInfo> entry : unsortedMachines.entrySet()) { + Ip value = entry.getKey(); MachineInfo key = entry.getValue(); map.put(key, value); logger.debug(location, jobid, "put: "+value); @@ -127,24 +124,14 @@ public class DuccMachinesData { String swapInuse = ""; String swapFree = ""; MachineInfo machineInfo = new MachineInfo(IDuccEnv.DUCC_NODES_FILE_PATH, "", nodeName, memTotal, memFree, swapInuse, swapFree, false, null, -1, 0); - unsortedMachines.put(machineInfo.getName(),machineInfo); + Ip machineIP = new Ip(machineInfo.getIp()); + unsortedMachines.put(machineIP,machineInfo); } updateSortedMachines(); } catch(Throwable t) { logger.warn(location, jobid, t); } - try { - InetAddress ia = InetAddress.getLocalHost(); - String chn = ia.getCanonicalHostName(); - int index = chn.indexOf("."); - if(index > 0) { - domain = chn.substring(index); - } - } - catch(Throwable t) { - logger.warn(location, jobid, t); - } } public boolean isPublished() { @@ -165,47 +152,23 @@ public class DuccMachinesData { } private void updateTotals(Ip ip, MachineSummaryInfo newInfo) { - if(summaryMachines.containsKey(ip.toString())) { - MachineSummaryInfo oldInfo = summaryMachines.get(ip.toString()); - summaryMachines.put(ip.toString(), newInfo); + if(summaryMachines.containsKey(ip)) { + MachineSummaryInfo oldInfo = summaryMachines.get(ip); + summaryMachines.put(ip, newInfo); memTotal.addAndGet(newInfo.memTotal-oldInfo.memTotal); memFree.addAndGet(newInfo.memFree-oldInfo.memFree); swapInuse.addAndGet(newInfo.swapInuse-oldInfo.swapInuse); swapFree.addAndGet(newInfo.swapFree-oldInfo.swapFree); } else { - summaryMachines.put(ip.toString(), newInfo); + summaryMachines.put(ip, newInfo); memTotal.addAndGet(newInfo.memTotal); memFree.addAndGet(newInfo.memFree); swapInuse.addAndGet(newInfo.swapInuse); swapFree.addAndGet(newInfo.swapFree); } } - - private String normalizeMachineName(String machineName) { - String retVal = machineName; - try { - if(!unsortedMachines.containsKey(machineName)) { - if(machineName.contains(".")) { - int index = machineName.indexOf("."); - String domainlessMachineName = machineName.substring(0,index); - if(unsortedMachines.containsKey(domainlessMachineName)) { - retVal = domainlessMachineName; - } - } - else { - String domainfullMachineName = machineName+domain; - if(unsortedMachines.containsKey(domainfullMachineName)) { - retVal = domainfullMachineName; - } - } - } - } - catch(Throwable t) { - } - return retVal; - } - + public void put(DatedNodeMetricsUpdateDuccEvent duccEvent) { String location = "put"; MachineSummaryInfo msi = new MachineSummaryInfo(); @@ -216,8 +179,9 @@ public class DuccMachinesData { ipToNodeUsersInfoMap.put(ip.toString(), map); } String machineName = nodeMetrics.getNodeIdentity().getName().trim(); - ipToNameMap.put(ip.toString(),machineName); - nameToIpMap.put(machineName,ip.toString()); + NodeId nodeId = new NodeId(machineName); + ipToNameMap.put(ip,nodeId); + nameToIpMap.put(nodeId,ip); // mem: total long nodeMemTotal = nodeMetrics.getNodeMemory().getMemTotal(); logger.debug(location, jobid, "node: "+machineName+" "+"memTotal: "+nodeMemTotal); @@ -252,7 +216,8 @@ public class DuccMachinesData { List<ProcessInfo> alienPids = nodeMetrics.getRogueProcessInfoList(); boolean cGroups = nodeMetrics.getCgroups(); MachineInfo current = new MachineInfo("", ip.toString(), machineName, memTotal, memFree, ""+swapInuse, ""+swapFree, cGroups, alienPids, duccEvent.getMillis(), duccEvent.getEventSize()); - String key = normalizeMachineName(machineName); + + Ip key = ip; MachineInfo previous = unsortedMachines.get(key); if(previous != null) { try { @@ -349,10 +314,14 @@ public class DuccMachinesData { return getPidCount(ip, user); } - public String getNameForIp(String ip) { + public String getNameForIp(String ipString) { String retVal = null; try { - retVal = ipToNameMap.get(ip); + Ip ip = new Ip(ipString); + NodeId nodeId = ipToNameMap.get(ip); + if(nodeId != null) { + retVal = nodeId.toString(); + } } catch(Throwable t) { } @@ -362,7 +331,11 @@ public class DuccMachinesData { public String getIpForName(String name) { String retVal = null; try { - retVal = nameToIpMap.get(name); + NodeId nodeId = new NodeId(name); + Ip ip = nameToIpMap.get(nodeId); + if(ip != null) { + retVal = ip.toString(); + } } catch(Throwable t) { } @@ -386,7 +359,7 @@ public class DuccMachinesData { public MachineFactsList getMachineFactsList() { Map<String, IDbMachine> dbMachineMap = DbQuery.getInstance().getMapMachines(); MachineFactsList factsList = new MachineFactsList(); - ConcurrentSkipListMap<MachineInfo,String> sortedMachines = getSortedMachines(); + ConcurrentSkipListMap<MachineInfo,Ip> sortedMachines = getSortedMachines(); Iterator<MachineInfo> iterator; iterator = sortedMachines.keySet().iterator(); while(iterator.hasNext()) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java Mon Feb 1 16:32:01 2016 @@ -103,6 +103,7 @@ import org.apache.uima.ducc.ws.registry. import org.apache.uima.ducc.ws.registry.sort.ServicesSortCache; import org.apache.uima.ducc.ws.server.IWebMonitor.MonitorType; import org.apache.uima.ducc.ws.sort.JobDetailsProcesses; +import org.apache.uima.ducc.ws.types.Ip; import org.apache.uima.ducc.ws.utils.FormatHelper; import org.apache.uima.ducc.ws.utils.FormatHelper.Precision; import org.apache.uima.ducc.ws.utils.HandlersHelper; @@ -3118,7 +3119,7 @@ public class DuccHandler extends DuccAbs sb.append("{ "); sb.append("\"aaData\": [ "); DuccMachinesData instance = DuccMachinesData.getInstance(); - ConcurrentSkipListMap<MachineInfo,String> sortedMachines = instance.getSortedMachines(); + ConcurrentSkipListMap<MachineInfo,Ip> sortedMachines = instance.getSortedMachines(); Iterator<MachineInfo> iterator; // pass 1 iterator = sortedMachines.keySet().iterator(); @@ -3532,10 +3533,10 @@ public class DuccHandler extends DuccAbs String cookie = DuccCookies.getCookie(request,DuccCookies.cookieAgents); if(cookie.equals(DuccCookies.valueAgentsShow)) { duccLogger.trace(methodName, jobid, "== show: "+cookie); - ConcurrentSkipListMap<String,MachineInfo> machines = DuccMachinesData.getInstance().getMachines(); - Iterator<String> iterator = machines.keySet().iterator(); + ConcurrentSkipListMap<Ip,MachineInfo> machines = DuccMachinesData.getInstance().getMachines(); + Iterator<Ip> iterator = machines.keySet().iterator(); while(iterator.hasNext()) { - String key = iterator.next(); + Ip key = iterator.next(); MachineInfo machineInfo = machines.get(key); Properties properties = DuccDaemonRuntimeProperties.getInstance().getAgent(machineInfo.getName()); if(first) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java Mon Feb 1 16:32:01 2016 @@ -83,6 +83,7 @@ import org.apache.uima.ducc.ws.registry. import org.apache.uima.ducc.ws.registry.sort.ServicesSortCache; import org.apache.uima.ducc.ws.server.DuccCookies.DisplayStyle; import org.apache.uima.ducc.ws.server.IWebMonitor.MonitorType; +import org.apache.uima.ducc.ws.types.Ip; import org.apache.uima.ducc.ws.types.NodeId; import org.apache.uima.ducc.ws.types.UserId; import org.apache.uima.ducc.ws.utils.FormatHelper.Precision; @@ -1473,10 +1474,10 @@ public class DuccHandlerClassic extends if(cookie.equals(DuccCookies.valueAgentsShow)) { duccLogger.trace(methodName, jobid, "== show: "+cookie); - ConcurrentSkipListMap<String,MachineInfo> machines = DuccMachinesData.getInstance().getMachines(); - Iterator<String> iterator = machines.keySet().iterator(); + ConcurrentSkipListMap<Ip,MachineInfo> machines = DuccMachinesData.getInstance().getMachines(); + Iterator<Ip> iterator = machines.keySet().iterator(); while(iterator.hasNext()) { - String key = iterator.next(); + Ip key = iterator.next(); MachineInfo machineInfo = machines.get(key); Properties properties = DuccDaemonRuntimeProperties.getInstance().getAgent(machineInfo.getName()); sb.append(trGet(counter)); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Mon Feb 1 16:32:01 2016 @@ -88,6 +88,7 @@ import org.apache.uima.ducc.ws.registry. import org.apache.uima.ducc.ws.registry.sort.ServicesSortCache; import org.apache.uima.ducc.ws.server.DuccCookies.DisplayStyle; import org.apache.uima.ducc.ws.server.IWebMonitor.MonitorType; +import org.apache.uima.ducc.ws.types.Ip; import org.apache.uima.ducc.ws.types.NodeId; import org.apache.uima.ducc.ws.types.UserId; import org.apache.uima.ducc.ws.utils.FormatHelper.Precision; @@ -1920,11 +1921,11 @@ public class DuccHandlerJsonFormat exten } if(showAgents) { - ConcurrentSkipListMap<String,MachineInfo> machines = DuccMachinesData.getInstance().getMachines(); - Iterator<String> iterator = machines.keySet().iterator(); + ConcurrentSkipListMap<Ip,MachineInfo> machines = DuccMachinesData.getInstance().getMachines(); + Iterator<Ip> iterator = machines.keySet().iterator(); while(iterator.hasNext()) { row = new JsonArray(); - String key = iterator.next(); + Ip key = iterator.next(); MachineInfo machineInfo = machines.get(key); Properties properties = DuccDaemonRuntimeProperties.getInstance().getAgent(machineInfo.getName()); // Status Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java Mon Feb 1 16:32:01 2016 @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentSkipListMap; import org.apache.uima.ducc.common.Node; @@ -52,6 +53,7 @@ import org.apache.uima.ducc.ws.DuccMachi import org.apache.uima.ducc.ws.IListenerOrchestrator; import org.apache.uima.ducc.ws.MachineInfo; import org.apache.uima.ducc.ws.server.DuccListeners; +import org.apache.uima.ducc.ws.types.Ip; public class NodeViz implements IListenerOrchestrator @@ -271,21 +273,19 @@ public class NodeViz } logger.debug(methodName, null, "Generateing visualizaiton"); - ConcurrentSkipListMap<String,MachineInfo> m = machineData.getMachines(); + ConcurrentSkipListMap<Ip,MachineInfo> m = machineData.getMachines(); - for (String s : m.keySet()) { - // + for (Entry<Ip, MachineInfo> entry : m.entrySet()) { + // // This is for hosts that have no work on them so they didn't come in the work map // - - MachineInfo mi = m.get(s); + MachineInfo mi = entry.getValue(); // NOTE: the map changes all the time so the value may be gone. This situation // will be fixed one day but for now just forget the node, it will show up // next time we get here. if ( mi == null ) continue; - + String s = mi.getName(); if ( !mi.getStatus().equals("up") ) continue; // filter non-up nodes - String key = strip(s); // our key, possibly with domain stripped if ( ! hosts.containsKey(key) ) { VisualizedHost vh = new VisualizedHost(mi, nc.getQuantumForNode(s)); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/Ip.java Mon Feb 1 16:32:01 2016 @@ -18,7 +18,7 @@ */ package org.apache.uima.ducc.ws.types; -public class Ip { +public class Ip implements Comparable<Ip> { private String ip; @@ -30,5 +30,30 @@ public class Ip { public String toString() { return this.ip; } - + + @Override + public int compareTo(Ip ip) { + int retVal = 0; + if(ip != null) { + Ip that = ip; + String thatIp = that.toString(); + String thisIp = this.toString(); + retVal = thisIp.compareTo(thatIp); + } + return retVal; + } + + @Override + public boolean equals(Object object) { + boolean retVal = false; + if(object != null) { + if(object instanceof Ip) { + Ip that = (Ip) object; + String thatIp = that.toString(); + String thisIp = this.toString(); + retVal = thisIp.equals(thatIp); + } + } + return retVal; + } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java?rev=1727969&r1=1727968&r2=1727969&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/types/NodeId.java Mon Feb 1 16:32:01 2016 @@ -18,7 +18,7 @@ */ package org.apache.uima.ducc.ws.types; -public class NodeId { +public class NodeId implements Comparable<NodeId> { private String machine; @@ -30,5 +30,30 @@ public class NodeId { public String toString() { return this.machine; } - + + @Override + public int compareTo(NodeId nodeId) { + int retVal = 0; + if(nodeId != null) { + NodeId that = nodeId; + String thatNodeId = that.toString(); + String thisNodeId = this.toString(); + retVal = thisNodeId.compareTo(thatNodeId); + } + return retVal; + } + + @Override + public boolean equals(Object object) { + boolean retVal = false; + if(object != null) { + if(object instanceof NodeId) { + NodeId that = (NodeId) object; + String thatNodeId = that.toString(); + String thisNodeId = this.toString(); + retVal = thisNodeId.equals(thatNodeId); + } + } + return retVal; + } }