Author: llu Date: Thu Dec 20 00:19:40 2012 New Revision: 1424242 URL: http://svn.apache.org/viewvc?rev=1424242&view=rev Log: YARN-223. Update process tree instead of getting new process trees. (Radim Kolar via llu)
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?rev=1424242&r1=1424241&r2=1424242&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Thu Dec 20 00:19:40 2012 @@ -61,8 +61,7 @@ import org.apache.hadoop.mapreduce.MRCon import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer; import org.apache.hadoop.mapreduce.task.ReduceContextImpl; -import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; -import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin.*; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.Progress; import org.apache.hadoop.util.Progressable; @@ -169,7 +168,7 @@ abstract public class Task implements Wr private Iterator<Long> currentRecIndexIterator = skipRanges.skipRangeIterator(); - private ResourceCalculatorPlugin resourceCalculator = null; + private ResourceCalculatorProcessTree pTree; private long initCpuCumulativeTime = 0; protected JobConf conf; @@ -372,7 +371,7 @@ abstract public class Task implements Wr * Return current state of the task. * needs to be synchronized as communication thread * sends the state every second - * @return + * @return task state */ synchronized TaskStatus.State getState(){ return this.taskStatus.getRunState(); @@ -558,15 +557,15 @@ abstract public class Task implements Wr } } committer.setupTask(taskContext); - Class<? extends ResourceCalculatorPlugin> clazz = - conf.getClass(MRConfig.RESOURCE_CALCULATOR_PLUGIN, - null, ResourceCalculatorPlugin.class); - resourceCalculator = ResourceCalculatorPlugin - .getResourceCalculatorPlugin(clazz, conf); - LOG.info(" Using ResourceCalculatorPlugin : " + resourceCalculator); - if (resourceCalculator != null) { - initCpuCumulativeTime = - resourceCalculator.getProcResourceValues().getCumulativeCpuTime(); + Class<? extends ResourceCalculatorProcessTree> clazz = + conf.getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE, + null, ResourceCalculatorProcessTree.class); + pTree = ResourceCalculatorProcessTree + .getResourceCalculatorProcessTree(System.getenv().get("JVM_PID"), clazz, conf); + LOG.info(" Using ResourceCalculatorProcessTree : " + pTree); + if (pTree != null) { + pTree.updateProcessTree(); + initCpuCumulativeTime = pTree.getCumulativeCpuTime(); } } @@ -817,14 +816,14 @@ abstract public class Task implements Wr // Update generic resource counters updateHeapUsageCounter(); - // Updating resources specified in ResourceCalculatorPlugin - if (resourceCalculator == null) { + // Updating resources specified in ResourceCalculatorProcessTree + if (pTree == null) { return; } - ProcResourceValues res = resourceCalculator.getProcResourceValues(); - long cpuTime = res.getCumulativeCpuTime(); - long pMem = res.getPhysicalMemorySize(); - long vMem = res.getVirtualMemorySize(); + pTree.updateProcessTree(); + long cpuTime = pTree.getCumulativeCpuTime(); + long pMem = pTree.getCumulativeRssmem(); + long vMem = pTree.getCumulativeVmem(); // Remove the CPU time consumed previously by JVM reuse cpuTime -= initCpuCumulativeTime; counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1424242&r1=1424241&r2=1424242&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Thu Dec 20 00:19:40 2012 @@ -55,8 +55,8 @@ public interface MRConfig { public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7*24*60*60*1000; // 7 days - public static final String RESOURCE_CALCULATOR_PLUGIN = - "mapreduce.job.resourcecalculatorplugin"; + public static final String RESOURCE_CALCULATOR_PROCESS_TREE = + "mapreduce.job.process-tree.class"; public static final String STATIC_RESOLUTIONS = "mapreduce.job.net.static.resolutions"; Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java?rev=1424242&r1=1424241&r2=1424242&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java Thu Dec 20 00:19:40 2012 @@ -409,7 +409,7 @@ public class LinuxResourceCalculatorPlug @Override public ProcResourceValues getProcResourceValues() { - pTree = pTree.getProcessTree(); + pTree.updateProcessTree(); long cpuTime = pTree.getCumulativeCpuTime(); long pMem = pTree.getCumulativeRssmem(); long vMem = pTree.getCumulativeVmem(); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java?rev=1424242&r1=1424241&r2=1424242&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java Thu Dec 20 00:19:40 2012 @@ -166,12 +166,10 @@ public class ProcfsBasedProcessTree exte } /** - * Get the process-tree with latest state. If the root-process is not alive, - * an empty tree will be returned. - * - * @return the process-tree with latest state. + * Update the process-tree with latest state. If the root-process is not alive, + * tree will become empty. */ - public ProcfsBasedProcessTree getProcessTree() { + public void updateProcessTree() { if (!pid.equals(deadPid)) { // Get the list of processes List<String> processList = getProcessList(); @@ -197,7 +195,7 @@ public class ProcfsBasedProcessTree exte } if (me == null) { - return this; + return; } // Add each process to its parent. @@ -239,7 +237,6 @@ public class ProcfsBasedProcessTree exte LOG.debug(this.toString()); } } - return this; } /** Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java?rev=1424242&r1=1424241&r2=1424242&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java Thu Dec 20 00:19:40 2012 @@ -151,7 +151,7 @@ public class TestProcfsBasedProcessTree ProcfsBasedProcessTree p = new ProcfsBasedProcessTree(pid, ProcessTree.isSetsidAvailable, ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL); - p = p.getProcessTree(); // initialize + p.updateProcessTree(); // initialize LOG.info("ProcessTree: " + p.toString()); File leaf = new File(lowestDescendant); @@ -164,7 +164,7 @@ public class TestProcfsBasedProcessTree } } - p = p.getProcessTree(); // reconstruct + p.updateProcessTree(); // reconstruct LOG.info("ProcessTree: " + p.toString()); // Get the process-tree dump @@ -203,7 +203,7 @@ public class TestProcfsBasedProcessTree } // ProcessTree is gone now. Any further calls should be sane. - p = p.getProcessTree(); + p.updateProcessTree(); assertFalse("ProcessTree must have been gone", p.isAlive()); assertTrue("Cumulative vmem for the gone-process is " + p.getCumulativeVmem() + " . It should be zero.", p @@ -336,7 +336,7 @@ public class TestProcfsBasedProcessTree new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir.getAbsolutePath()); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // verify cumulative memory assertEquals("Cumulative virtual memory does not match", 600000L, @@ -362,7 +362,7 @@ public class TestProcfsBasedProcessTree writeStatFiles(procfsRootDir, pids, procInfos); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // verify cumulative cpu time again cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ? @@ -409,7 +409,7 @@ public class TestProcfsBasedProcessTree new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir.getAbsolutePath()); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // verify cumulative memory assertEquals("Cumulative memory does not match", @@ -425,7 +425,7 @@ public class TestProcfsBasedProcessTree writeStatFiles(procfsRootDir, newPids, newProcInfos); // check memory includes the new process. - processTree.getProcessTree(); + processTree.updateProcessTree(); assertEquals("Cumulative vmem does not include new process", 1200000L, processTree.getCumulativeVmem()); long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? @@ -451,7 +451,7 @@ public class TestProcfsBasedProcessTree writeStatFiles(procfsRootDir, newPids, newProcInfos); // refresh process tree - processTree.getProcessTree(); + processTree.updateProcessTree(); // processes older than 2 iterations should be same as before. assertEquals("Cumulative vmem shouldn't have included new processes", @@ -555,7 +555,7 @@ public class TestProcfsBasedProcessTree new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir .getAbsolutePath()); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // Get the process-tree dump String processTreeDump = processTree.getProcessTreeDump();