Author: arp Date: Fri Sep 13 21:27:58 2013 New Revision: 1523110 URL: http://svn.apache.org/r1523110 Log: Merging r1521566 through r1523108 from trunk to branch HDFS-2832
Added: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppComponentDependencies.java - copied unchanged from r1523108, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppComponentDependencies.java Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred.cmd hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed) Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1521566-1523108 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt Fri Sep 13 21:27:58 2013 @@ -186,6 +186,10 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5478. TeraInputFormat unnecessarily defines its own FileSplit subclass (Sandy Ryza) + MAPREDUCE-5497. Changed MRAppMaster to sleep only after doing everything else + but just before ClientService to avoid race conditions during RM restart. + (Jian He via vinodkv) + OPTIMIZATIONS MAPREDUCE-5446. TestJobHistoryEvents and TestJobHistoryParsing have race @@ -256,6 +260,11 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5414. TestTaskAttempt fails in JDK7 with NPE (Nemon Lou via devaraj) + MAPREDUCE-5020. Compile failure with JDK8 (Trevor Robinson via tgraves) + + MAPREDUCE-5164. mapred job and queue commands omit HADOOP_CLIENT_OPTS + (Nemon Lou via devaraj) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1521566-1523108 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred Fri Sep 13 21:27:58 2013 @@ -61,8 +61,10 @@ esac if [ "$COMMAND" = "job" ] ; then CLASS=org.apache.hadoop.mapred.JobClient + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "queue" ] ; then CLASS=org.apache.hadoop.mapred.JobQueueClient + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "pipes" ] ; then CLASS=org.apache.hadoop.mapred.pipes.Submitter HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred.cmd URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred.cmd?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred.cmd (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/bin/mapred.cmd Fri Sep 13 21:27:58 2013 @@ -103,10 +103,12 @@ goto :eof :job set CLASS=org.apache.hadoop.mapred.JobClient + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% goto :eof :queue set CLASS=org.apache.hadoop.mapred.JobQueueClient + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% goto :eof :sampler Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1521566-1523108 Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Fri Sep 13 21:27:58 2013 @@ -362,7 +362,10 @@ public class MRAppMaster extends Composi //service to handle requests from JobClient clientService = createClientService(context); - addIfService(clientService); + // Init ClientService separately so that we stop it separately, since this + // service needs to wait some time before it stops so clients can know the + // final states + clientService.init(conf); containerAllocator = createContainerAllocator(clientService, context); @@ -425,7 +428,6 @@ public class MRAppMaster extends Composi // queued inside the JobHistoryEventHandler addIfService(historyService); } - super.serviceInit(conf); } // end of init() @@ -534,14 +536,6 @@ public class MRAppMaster extends Composi } } - // TODO:currently just wait for some time so clients can know the - // final states. Will be removed once RM come on. - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - try { //if isLastAMRetry comes as true, should never set it to false if ( !isLastAMRetry){ @@ -556,6 +550,14 @@ public class MRAppMaster extends Composi LOG.info("Calling stop for all the services"); MRAppMaster.this.stop(); + // TODO: Stop ClientService last, since only ClientService should wait for + // some time so clients can know the final states. Will be removed once RM come on. + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + clientService.stop(); } catch (Throwable t) { LOG.warn("Graceful stop failed ", t); } @@ -1019,8 +1021,10 @@ public class MRAppMaster extends Composi LOG.info("MRAppMaster launching normal, non-uberized, multi-container " + "job " + job.getID() + "."); } + // Start ClientService here, since it's not initialized if + // errorHappenedShutDown is true + clientService.start(); } - //start all the components super.serviceStart(); Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ClientService.java Fri Sep 13 21:27:58 2013 @@ -1,28 +1,30 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.mapreduce.v2.app.client; import java.net.InetSocketAddress; -public interface ClientService { +import org.apache.hadoop.service.Service; - InetSocketAddress getBindAddress(); +public interface ClientService extends Service { - int getHttpPort(); + public abstract InetSocketAddress getBindAddress(); + + public abstract int getHttpPort(); } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Fri Sep 13 21:27:58 2013 @@ -94,8 +94,7 @@ import org.apache.hadoop.yarn.webapp.Web * jobclient (user facing). * */ -public class MRClientService extends AbstractService - implements ClientService { +public class MRClientService extends AbstractService implements ClientService { static final Log LOG = LogFactory.getLog(MRClientService.class); @@ -106,7 +105,7 @@ public class MRClientService extends Abs private AppContext appContext; public MRClientService(AppContext appContext) { - super("MRClientService"); + super(MRClientService.class.getName()); this.appContext = appContext; this.protocolHandler = new MRClientProtocolHandler(); } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Fri Sep 13 21:27:58 2013 @@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.records.TaskReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; +import org.apache.hadoop.mapreduce.v2.app.client.MRClientService; import org.apache.hadoop.mapreduce.v2.app.commit.CommitterEvent; import org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler; import org.apache.hadoop.mapreduce.v2.app.job.Job; @@ -603,7 +604,7 @@ public class MRApp extends MRAppMaster { @Override protected ClientService createClientService(AppContext context) { - return new ClientService(){ + return new MRClientService(context) { @Override public InetSocketAddress getBindAddress() { return NetUtils.createSocketAddr("localhost:9876"); Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Fri Sep 13 21:27:58 2013 @@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.jobhi import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; +import org.apache.hadoop.mapreduce.v2.app.client.MRClientService; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent; @@ -284,14 +285,12 @@ import org.junit.Test; private final class MRAppTestCleanup extends MRApp { int stagingDirCleanedup; int ContainerAllocatorStopped; - int JobHistoryEventHandlerStopped; int numStops; public MRAppTestCleanup(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart) { super(maps, reduces, autoComplete, testName, cleanOnStart); stagingDirCleanedup = 0; ContainerAllocatorStopped = 0; - JobHistoryEventHandlerStopped = 0; numStops = 0; } @@ -319,26 +318,6 @@ import org.junit.Test; } @Override - protected EventHandler<JobHistoryEvent> createJobHistoryHandler( - AppContext context) { - return new TestJobHistoryEventHandler(context, getStartCount()); - } - - private class TestJobHistoryEventHandler extends JobHistoryEventHandler { - - public TestJobHistoryEventHandler(AppContext context, int startCount) { - super(context, startCount); - } - - @Override - public void serviceStop() throws Exception { - numStops++; - JobHistoryEventHandlerStopped = numStops; - super.serviceStop(); - } - } - - @Override protected ContainerAllocator createContainerAllocator( ClientService clientService, AppContext context) { return new TestCleanupContainerAllocator(); @@ -405,15 +384,13 @@ import org.junit.Test; app.verifyCompleted(); int waitTime = 20 * 1000; - while (waitTime > 0 && app.numStops < 3 ) { + while (waitTime > 0 && app.numStops < 2) { Thread.sleep(100); waitTime -= 100; } - // assert JobHistoryEventHandlerStopped first, then - // ContainerAllocatorStopped, and then stagingDirCleanedup - Assert.assertEquals(1, app.JobHistoryEventHandlerStopped); - Assert.assertEquals(2, app.ContainerAllocatorStopped); - Assert.assertEquals(3, app.stagingDirCleanedup); + // assert ContainerAllocatorStopped and then tagingDirCleanedup + Assert.assertEquals(1, app.ContainerAllocatorStopped); + Assert.assertEquals(2, app.stagingDirCleanedup); } } Modified: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java?rev=1523110&r1=1523109&r2=1523110&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/InputSampler.java Fri Sep 13 21:27:58 2013 @@ -317,7 +317,7 @@ public class InputSampler<K,V> extends C final InputFormat inf = ReflectionUtils.newInstance(job.getInputFormatClass(), conf); int numPartitions = job.getNumReduceTasks(); - K[] samples = sampler.getSample(inf, job); + K[] samples = (K[])sampler.getSample(inf, job); LOG.info("Using " + samples.length + " samples"); RawComparator<K> comparator = (RawComparator<K>) job.getSortComparator(); Propchange: hadoop/common/branches/HDFS-2832/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1521566-1523108