Author: cnauroth
Date: Tue Apr 22 19:32:17 2014
New Revision: 1589259
URL: http://svn.apache.org/r1589259
Log:
MAPREDUCE-5833. TestRMContainerAllocator fails ocassionally. Contributed by
Zhijie Shen.
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1589259&r1=1589258&r2=1589259&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue
Apr 22 19:32:17 2014
@@ -79,6 +79,9 @@ Release 2.4.1 - UNRELEASED
MAPREDUCE-5827. TestSpeculativeExecutionWithMRApp fails.
(Zhijie Shen via cnauroth)
+ MAPREDUCE-5833. TestRMContainerAllocator fails ocassionally.
+ (Zhijie Shen via cnauroth)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1589259&r1=1589258&r2=1589259&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Tue Apr 22 19:32:17 2014
@@ -151,7 +151,8 @@ public class RMContainerAllocator extend
private long retryInterval;
private long retrystartTime;
- BlockingQueue<ContainerAllocatorEvent> eventQueue
+ @VisibleForTesting
+ protected BlockingQueue<ContainerAllocatorEvent> eventQueue
= new LinkedBlockingQueue<ContainerAllocatorEvent>();
private ScheduleStats scheduleStats = new ScheduleStats();
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1589259&r1=1589258&r2=1589259&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
Tue Apr 22 19:32:17 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
@@ -62,6 +63,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
@@ -73,6 +75,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -104,6 +107,8 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.base.Supplier;
+
@SuppressWarnings("unchecked")
public class TestRMContainerAllocator {
@@ -579,7 +584,7 @@ public class TestRMContainerAllocator {
MyContainerAllocator allocator = (MyContainerAllocator) mrApp
.getContainerAllocator();
- mrApp.waitForState(job, JobState.RUNNING);
+ mrApp.waitForInternalState((JobImpl) job, JobStateInternal.RUNNING);
amDispatcher.await();
// Wait till all map-attempts request for containers
@@ -731,7 +736,7 @@ public class TestRMContainerAllocator {
MyContainerAllocator allocator = (MyContainerAllocator) mrApp
.getContainerAllocator();
- mrApp.waitForState(job, JobState.RUNNING);
+ mrApp.waitForInternalState((JobImpl)job, JobStateInternal.RUNNING);
amDispatcher.await();
// Wait till all map-attempts request for containers
@@ -1550,7 +1555,15 @@ public class TestRMContainerAllocator {
}
// API to be used by tests
- public List<TaskAttemptContainerAssignedEvent> schedule() {
+ public List<TaskAttemptContainerAssignedEvent> schedule()
+ throws Exception {
+ // before doing heartbeat with RM, drain all the outstanding events to
+ // ensure all the requests before this heartbeat is to be handled
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ return eventQueue.isEmpty();
+ }
+ }, 100, 10000);
// run the scheduler
try {
super.heartbeat();
@@ -1586,7 +1599,6 @@ public class TestRMContainerAllocator {
public boolean isUnregistered() {
return isUnregistered;
}
-
}
@Test
