This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new 57e88a6 YARN-9552. FairScheduler: NODE_UPDATE can cause NoSuchElementException. Contributed by Peter Bacsko. 57e88a6 is described below commit 57e88a63cfb6d50507c92f268b36f3b23debe7e5 Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Wed Oct 9 14:19:56 2019 +0200 YARN-9552. FairScheduler: NODE_UPDATE can cause NoSuchElementException. Contributed by Peter Bacsko. --- .../scheduler/AppSchedulingInfo.java | 11 +++--- .../scheduler/fair/FSAppAttempt.java | 4 +-- .../scheduler/fair/TestFSAppAttempt.java | 42 ++++++++++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8074f06..21f052a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -477,14 +477,17 @@ public class AppSchedulingInfo { } public PendingAsk getNextPendingAsk() { + readLock.lock(); try { - readLock.lock(); - SchedulerRequestKey firstRequestKey = schedulerKeys.first(); - return getPendingAsk(firstRequestKey, ResourceRequest.ANY); + if (!schedulerKeys.isEmpty()) { + SchedulerRequestKey firstRequestKey = schedulerKeys.first(); + return getPendingAsk(firstRequestKey, ResourceRequest.ANY); + } else { + return null; + } } finally { readLock.unlock(); } - } public PendingAsk getPendingAsk(SchedulerRequestKey schedulerKey) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 2355fb7..0eedec0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -936,8 +936,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt if (!isAmRunning() && !getUnmanagedAM()) { // Return true if we have not ask, or queue is not be able to run app's AM PendingAsk ask = appSchedulingInfo.getNextPendingAsk(); - if (ask.getCount() == 0 || !getQueue().canRunAppAM( - ask.getPerAllocationResource())) { + if (ask != null && (ask.getCount() == 0 || !getQueue().canRunAppAM( + ask.getPerAllocationResource()))) { return true; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java index 51ffd23..ad00a1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java @@ -19,7 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; @@ -33,8 +36,12 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -341,6 +348,41 @@ public class TestFSAppAttempt extends FairSchedulerTestBase { assertEquals(clusterResource, spyApp.getHeadroom()); } + /** + * Ensure that no pending ask request inside appSchedulingInfo + * does not result in an error. + */ + @Test + public void testNoNextPendingAsk() { + FSLeafQueue queue = Mockito.mock(FSLeafQueue.class); + ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + RMContext rmContext = Mockito.mock(RMContext.class); + ConcurrentMap<ApplicationId, RMApp> rmApps = new ConcurrentHashMap<>(); + RMApp rmApp = Mockito.mock(RMApp.class); + rmApps.put(applicationAttemptId.getApplicationId(), rmApp); + ApplicationSubmissionContext appContext = + Mockito.mock(ApplicationSubmissionContext.class); + Mockito.when(appContext.getUnmanagedAM()).thenReturn(false); + LogAggregationContext logAggregationContext = + Mockito.mock(LogAggregationContext.class); + Mockito.when(appContext.getLogAggregationContext()) + .thenReturn(logAggregationContext); + Mockito.when(rmApp.getApplicationSchedulingEnvs()) + .thenReturn(new HashMap<>()); + Mockito.when(rmApp.getApplicationSubmissionContext()) + .thenReturn(appContext); + Mockito.when(rmContext.getRMApps()).thenReturn(rmApps); + FSAppAttempt schedulerApp = + new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, + null, rmContext); + schedulerApp.setAmRunning(false); + FSSchedulerNode schedulerNode = Mockito.mock(FSSchedulerNode.class); + + Resource resource = schedulerApp.assignContainer(schedulerNode); + + assertEquals(Resources.none(), resource); + } + private static long min(long value1, long value2, long value3) { return Math.min(Math.min(value1, value2), value3); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org