Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java Fri Jan 11 19:40:23 2013 @@ -31,6 +31,11 @@ public interface MiniMRClientCluster { public void start() throws IOException; + /** + * Stop and start back the cluster using the same configuration. + */ + public void restart() throws IOException; + public void stop() throws IOException; public Configuration getConfig() throws IOException;
Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java Fri Jan 11 19:40:23 2013 @@ -67,6 +67,10 @@ public class MiniMRClientClusterFactory MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller .getName(), noOfNMs); + job.getConfiguration().set("minimrclientcluster.caller.name", + caller.getName()); + job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", + noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.start(); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java Fri Jan 11 19:40:23 2013 @@ -18,8 +18,13 @@ package org.apache.hadoop.mapred; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.Service.STATE; /** * An adapter for MiniMRYarnCluster providing a MiniMRClientCluster interface. @@ -29,6 +34,8 @@ public class MiniMRYarnClusterAdapter im private MiniMRYarnCluster miniMRYarnCluster; + private static final Log LOG = LogFactory.getLog(MiniMRYarnClusterAdapter.class); + public MiniMRYarnClusterAdapter(MiniMRYarnCluster miniMRYarnCluster) { this.miniMRYarnCluster = miniMRYarnCluster; } @@ -48,4 +55,22 @@ public class MiniMRYarnClusterAdapter im miniMRYarnCluster.stop(); } + @Override + public void restart() { + if (!miniMRYarnCluster.getServiceState().equals(STATE.STARTED)){ + LOG.warn("Cannot restart the mini cluster, start it first"); + return; + } + Configuration oldConf = new Configuration(getConfig()); + String callerName = oldConf.get("minimrclientcluster.caller.name", + this.getClass().getName()); + int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1); + oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); + stop(); + miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs); + miniMRYarnCluster.init(oldConf); + miniMRYarnCluster.start(); + } + } Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java Fri Jan 11 19:40:23 2013 @@ -32,6 +32,8 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -92,6 +94,65 @@ public class TestMiniMRClientCluster { } @Test + public void testRestart() throws Exception { + + String rmAddress1 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); + String rmAdminAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_ADMIN_ADDRESS); + String rmSchedAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_SCHEDULER_ADDRESS); + String rmRstrackerAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + String rmWebAppAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_WEBAPP_ADDRESS); + + String mrHistAddress1 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_ADDRESS); + String mrHistWebAppAddress1 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + + mrCluster.restart(); + + String rmAddress2 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); + String rmAdminAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_ADMIN_ADDRESS); + String rmSchedAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_SCHEDULER_ADDRESS); + String rmRstrackerAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + String rmWebAppAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_WEBAPP_ADDRESS); + + String mrHistAddress2 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_ADDRESS); + String mrHistWebAppAddress2 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + + assertEquals("Address before restart: " + rmAddress1 + + " is different from new address: " + rmAddress2, rmAddress1, + rmAddress2); + assertEquals("Address before restart: " + rmAdminAddress1 + + " is different from new address: " + rmAdminAddress2, + rmAdminAddress1, rmAdminAddress2); + assertEquals("Address before restart: " + rmSchedAddress1 + + " is different from new address: " + rmSchedAddress2, + rmSchedAddress1, rmSchedAddress2); + assertEquals("Address before restart: " + rmRstrackerAddress1 + + " is different from new address: " + rmRstrackerAddress2, + rmRstrackerAddress1, rmRstrackerAddress2); + assertEquals("Address before restart: " + rmWebAppAddress1 + + " is different from new address: " + rmWebAppAddress2, + rmWebAppAddress1, rmWebAppAddress2); + assertEquals("Address before restart: " + mrHistAddress1 + + " is different from new address: " + mrHistAddress2, mrHistAddress1, + mrHistAddress2); + assertEquals("Address before restart: " + mrHistWebAppAddress1 + + " is different from new address: " + mrHistWebAppAddress2, + mrHistWebAppAddress1, mrHistWebAppAddress2); + + } + + @Test public void testJob() throws Exception { final Job job = createJob(); org.apache.hadoop.mapreduce.lib.input.FileInputFormat.setInputPaths(job, Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java Fri Jan 11 19:40:23 2013 @@ -45,7 +45,7 @@ public class TestJobOutputCommitter exte private static String TEST_ROOT_DIR = new File(System.getProperty( "test.build.data", "/tmp") - + "/" + "test-job-cleanup").toString(); + + "/" + "test-job-output-committer").toString(); private static final String CUSTOM_CLEANUP_FILE_NAME = "_custom_cleanup"; private static final String ABORT_KILLED_FILE_NAME = "_custom_abort_killed"; private static final String ABORT_FAILED_FILE_NAME = "_custom_abort_failed"; Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java Fri Jan 11 19:40:23 2013 @@ -151,7 +151,7 @@ public class TestProcfsBasedProcessTree ProcfsBasedProcessTree p = new ProcfsBasedProcessTree(pid, ProcessTree.isSetsidAvailable, ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL); - p = p.getProcessTree(); // initialize + p.updateProcessTree(); // initialize LOG.info("ProcessTree: " + p.toString()); File leaf = new File(lowestDescendant); @@ -164,7 +164,7 @@ public class TestProcfsBasedProcessTree } } - p = p.getProcessTree(); // reconstruct + p.updateProcessTree(); // reconstruct LOG.info("ProcessTree: " + p.toString()); // Get the process-tree dump @@ -203,7 +203,7 @@ public class TestProcfsBasedProcessTree } // ProcessTree is gone now. Any further calls should be sane. - p = p.getProcessTree(); + p.updateProcessTree(); assertFalse("ProcessTree must have been gone", p.isAlive()); assertTrue("Cumulative vmem for the gone-process is " + p.getCumulativeVmem() + " . It should be zero.", p @@ -336,7 +336,7 @@ public class TestProcfsBasedProcessTree new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir.getAbsolutePath()); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // verify cumulative memory assertEquals("Cumulative virtual memory does not match", 600000L, @@ -362,7 +362,7 @@ public class TestProcfsBasedProcessTree writeStatFiles(procfsRootDir, pids, procInfos); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // verify cumulative cpu time again cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ? @@ -409,7 +409,7 @@ public class TestProcfsBasedProcessTree new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir.getAbsolutePath()); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // verify cumulative memory assertEquals("Cumulative memory does not match", @@ -425,7 +425,7 @@ public class TestProcfsBasedProcessTree writeStatFiles(procfsRootDir, newPids, newProcInfos); // check memory includes the new process. - processTree.getProcessTree(); + processTree.updateProcessTree(); assertEquals("Cumulative vmem does not include new process", 1200000L, processTree.getCumulativeVmem()); long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? @@ -451,7 +451,7 @@ public class TestProcfsBasedProcessTree writeStatFiles(procfsRootDir, newPids, newProcInfos); // refresh process tree - processTree.getProcessTree(); + processTree.updateProcessTree(); // processes older than 2 iterations should be same as before. assertEquals("Cumulative vmem shouldn't have included new processes", @@ -555,7 +555,7 @@ public class TestProcfsBasedProcessTree new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir .getAbsolutePath()); // build the process tree. - processTree.getProcessTree(); + processTree.updateProcessTree(); // Get the process-tree dump String processTreeDump = processTree.getProcessTreeDump(); Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml?rev=1432246&r1=1432245&r2=1432246&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml Fri Jan 11 19:40:23 2013 @@ -194,5 +194,6 @@ <module>hadoop-mapreduce-client-app</module> <module>hadoop-mapreduce-client-jobclient</module> <module>hadoop-mapreduce-client-hs</module> + <module>hadoop-mapreduce-client-hs-plugins</module> </modules> </project>
