Author: challngr Date: Wed Sep 23 17:50:44 2015 New Revision: 1704906 URL: http://svn.apache.org/viewvc?rev=1704906&view=rev Log: UIMA-4569 Supply per-node quantum from NodeConfiguration.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1704906&r1=1704905&r2=1704906&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java Wed Sep 23 17:50:44 2015 @@ -65,14 +65,16 @@ public class NodeConfiguration Map<String, DuccProperties> usermap = new HashMap<String, DuccProperties>(); // all users, by name UIMA-4275 ArrayList<String> independentClasses = new ArrayList<String>(); // all classes that don't derive from something - Map<String, String> allNodes = new HashMap<String, String>(); // map node -> nodepool name, map for dup checking + Map<String, String> allNodes = new HashMap<String, String>(); // map node -> nodefile name, map for dup checking Map<String, DuccProperties> poolsByNodefile = new HashMap<String, DuccProperties>(); // nodepool node file -> nodepool props + Map<String, DuccProperties> poolsByNodeName = new HashMap<String, DuccProperties>(); // Nodepools, by node Map<String, String> allImports = new HashMap<String, String>(); // map nodefile -> importer, map for dup checking Map<String, String> referrers = new HashMap<String, String>(); // map nodefile -> referring nodepool, for dup checking DuccLogger logger; String defaultDomain = null; + int defaultQuantum = 15; String firstNodepool = null; boolean fairShareExists = false; @@ -651,7 +653,7 @@ public class NodeConfiguration for (DuccProperties props : independentNodepools ) { String q = props.getProperty("share-quantum"); if ( q == null ) { - props.setProperty("share-quantum", ""+SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", 15)); + props.setProperty("share-quantum", "" + defaultQuantum); } else { try { Integer.parseInt(q); // insure it's a number @@ -797,6 +799,7 @@ public class NodeConfiguration } allNodes.put(node, nodefile); // for dup checking - we only get to read a node once nodes.put(node, nodefile); // UIMA-4142 map host -> domain + poolsByNodeName.put(node, p); // So we can find pool-related things for the node // include fully and non-fully qualified names to allow sloppiness of config ndx = node.indexOf("."); @@ -998,7 +1001,18 @@ public class NodeConfiguration return reserveDefault; } - public int getShareQuantum(String classname) + public int getQuantumForNode(String node) + { + DuccProperties np = poolsByNodeName.get(node); + if ( np == null ) { + // it has to be the default np, so use the default quantum + return defaultQuantum; + } + // otherwise it's required that the quantum for the nodepool is set to something + return Integer.parseInt(np.getProperty("share-quantum")); + } + + public int getQuantumForClass(String classname) throws IllegalConfigurationException { // to find the quantum for a class - @@ -1051,6 +1065,7 @@ public class NodeConfiguration throw new IllegalConfigurationException("DUCC_HOME must be defined as a system property."); } defaultDomain = getDomainName(); + defaultQuantum = SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", 15); try { String tconfig_file_name = resolve(config_file_name); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1704906&r1=1704905&r2=1704906&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Wed Sep 23 17:50:44 2015 @@ -18,10 +18,7 @@ */ package org.apache.uima.ducc.ws; -import java.util.ArrayList; import java.util.List; -import java.util.ListIterator; -import java.util.TreeMap; import org.apache.uima.ducc.common.IDuccEnv; import org.apache.uima.ducc.common.config.CommonConfiguration; @@ -87,23 +84,6 @@ public class DuccBoot extends Thread { return limit; } - private static TreeMap<Integer,String> sort(ArrayList<String> list) { - String location = "sort"; - TreeMap<Integer,String> map = new TreeMap<Integer,String>(); - ListIterator<String> listIterator = list.listIterator(); - while(listIterator.hasNext()) { - try { - String value = listIterator.next().trim(); - int index = value.indexOf('.'); - Integer key = Integer.parseInt(value.substring(0,index)); - map.put(key, value); - } - catch(Throwable t) { - logger.warn(location, jobid, t); - } - } - return map; - } // private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) { // String location = "restoreReservations"; Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java?rev=1704906&r1=1704905&r2=1704906&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java Wed Sep 23 17:50:44 2015 @@ -19,8 +19,6 @@ package org.apache.uima.ducc.ws.server.nodeviz; -import java.io.FileNotFoundException; -import java.io.IOException; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; @@ -33,7 +31,6 @@ import org.apache.uima.ducc.common.SizeB import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.DuccLoggerComponents; import org.apache.uima.ducc.common.utils.DuccPropertiesResolver; -import org.apache.uima.ducc.common.utils.IllegalConfigurationException; import org.apache.uima.ducc.common.utils.SystemPropertyResolver; import org.apache.uima.ducc.common.utils.Version; import org.apache.uima.ducc.transport.event.OrchestratorStateDuccEvent; @@ -125,11 +122,6 @@ public class NodeViz IDuccWorkMap jobmap = ev.getWorkMap(); - int job_shares = 0; - int service_shares = 0; - int pop_shares = 0; - int reservation_shares = 0; - int job_gb = 0; int service_gb = 0; int pop_gb = 0; @@ -189,7 +181,7 @@ public class NodeViz String sclass = sti.getSchedulingClass(); int quantum = default_quantum; try { - quantum = nc.getShareQuantum(sclass); + quantum = nc.getQuantumForClass(sclass); } catch ( Exception e ) { // this most likely caused by a reconfigure so that a job's class no longer exists. nothing to do about it // but punt and try not to crash. @@ -219,15 +211,12 @@ public class NodeViz switch ( type ) { case Job: - job_shares += qshares; job_gb += jobmem; break; - case Pop: - pop_shares += qshares; + case Pop: pop_gb += jobmem; break; - case Service: - service_shares += qshares; + case Service: service_gb += jobmem; break; } @@ -253,7 +242,6 @@ public class NodeViz IDuccReservationMap rm = de.getReservationMap(); logger.debug(methodName, w.getDuccId(), "Receive:", type, w.getStateObject(), "processes[", rm.size(), "] Completed:", w.isCompleted()); - reservation_shares += qshares; reservation_gb += jobmem; for ( IDuccReservation r: rm.values()) { @@ -298,9 +286,7 @@ public class NodeViz String key = strip(s); // our key, possibly with domain stripped if ( ! hosts.containsKey(key) ) { - // System.out.println("Set host from MachineInfo with key :" + key + ":"); - - VisualizedHost vh = new VisualizedHost(mi, default_quantum); + VisualizedHost vh = new VisualizedHost(mi, nc.getQuantumForNode(s)); hosts.put(key, vh); } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java?rev=1704906&r1=1704905&r2=1704906&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java Wed Sep 23 17:50:44 2015 @@ -77,18 +77,9 @@ class VisualizedHost this.name = NodeViz.strip(info.getName()); this.ip = info.getIp(); - //TODO - String ns = ""; - if ( false && ns == "" || ns == null ) { - this.mem = 0; - this.shares = 0; - this.mem_reservable = 0; - } else { - this.mem = Integer.parseInt(info.getMemFree()); - this.shares = (mem / quantum); - this.mem_reservable = shares * quantum; - } - + this.mem = Integer.parseInt(info.getMemFree()); + this.shares = (mem / quantum); + this.mem_reservable = shares * quantum; this.shares_free = shares; }