[ https://issues.apache.org/jira/browse/YARN-6163?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15862478#comment-15862478 ]
ASF GitHub Bot commented on YARN-6163: -------------------------------------- Github user kambatla commented on a diff in the pull request: https://github.com/apache/hadoop/pull/192#discussion_r100673331 --- Diff: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java --- @@ -220,6 +220,62 @@ public void updateInternal(boolean checkStarvation) { } /** + * Compute the extent of fairshare starvation for a set of apps. + * + * @param appsWithDemand apps to compute fairshare starvation for + * @return aggregate fairshare starvation for all apps + */ + private Resource updateStarvedAppsFairshare( + TreeSet<FSAppAttempt> appsWithDemand) { + Resource fairShareStarvation = Resources.clone(none()); + // Fetch apps with unmet demand sorted by fairshare starvation + for (FSAppAttempt app : appsWithDemand) { + Resource appStarvation = app.fairShareStarvation(); + if (!Resources.isNone(appStarvation)) { + context.getStarvedApps().addStarvedApp(app); + Resources.addTo(fairShareStarvation, appStarvation); + } else { + break; + } + } + return fairShareStarvation; + } + + /** + * Distribute minshare starvation to a set of apps + * @param appsWithDemand set of apps + * @param minShareStarvation minshare starvation to distribute + */ + private void updateStarvedAppsMinshare( + TreeSet<FSAppAttempt> appsWithDemand, Resource minShareStarvation) { + // Keep adding apps to the starved list until the unmet demand goes over + // the remaining minshare + for (FSAppAttempt app : appsWithDemand) { + if (!Resources.isNone(minShareStarvation())) { + Resource appMinShare = app.getPendingDemand(); + Resources.subtractFromNonNegative( + appMinShare, app.getFairshareStarvation()); + + if (Resources.greaterThan(policy.getResourceCalculator(), + scheduler.getClusterResource(), + appMinShare, minShareStarvation)) { + Resources.subtractFromNonNegative( + appMinShare, minShareStarvation); + minShareStarvation = none(); --- End diff -- Was avoiding cloning the object, but agree that is error-prone. Updated. > FS Preemption is a trickle for severely starved applications > ------------------------------------------------------------ > > Key: YARN-6163 > URL: https://issues.apache.org/jira/browse/YARN-6163 > Project: Hadoop YARN > Issue Type: Sub-task > Components: fairscheduler > Affects Versions: 2.9.0 > Reporter: Karthik Kambatla > Assignee: Karthik Kambatla > Attachments: yarn-6163-1.patch > > > With current logic, only one RR is considered per each instance of marking an > application starved. This marking happens only on the update call that runs > every 500ms. Due to this, an application that is severely starved takes > forever to reach fairshare based on preemptions. -- This message was sent by Atlassian JIRA (v6.3.15#6346) --------------------------------------------------------------------- To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org