YARN-5865. Retrospect updateApplicationPriority api to handle state store exception in align with YARN-5611. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a926f895 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a926f895 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a926f895 Branch: refs/heads/YARN-2915 Commit: a926f895c11cd69cf2117c3b970304f3f1f53d92 Parents: 6f80742 Author: Rohith Sharma K S <rohithsharm...@apache.org> Authored: Tue Nov 22 14:49:15 2016 +0530 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Tue Nov 22 14:49:15 2016 +0530 ---------------------------------------------------------------------- .../ApplicationMasterService.java | 2 +- .../server/resourcemanager/ClientRMService.java | 10 ++-- .../server/resourcemanager/RMAppManager.java | 49 ++++++++++++++++++-- .../metrics/TimelineServiceV1Publisher.java | 4 +- .../metrics/TimelineServiceV2Publisher.java | 4 +- .../server/resourcemanager/rmapp/RMApp.java | 14 ++++++ .../server/resourcemanager/rmapp/RMAppImpl.java | 37 ++++++++++++--- .../scheduler/AbstractYarnScheduler.java | 6 ++- .../scheduler/YarnScheduler.java | 11 ++++- .../scheduler/capacity/CapacityScheduler.java | 28 +++++------ .../scheduler/event/AppAddedSchedulerEvent.java | 5 +- .../resourcemanager/webapp/RMWebServices.java | 6 +-- .../resourcemanager/webapp/dao/AppInfo.java | 9 ++-- .../server/resourcemanager/TestAppManager.java | 1 + .../TestApplicationMasterService.java | 6 +-- .../resourcemanager/TestClientRMService.java | 13 ++++-- .../applicationsmanager/MockAsm.java | 11 +++++ .../metrics/TestSystemMetricsPublisher.java | 2 + .../TestSystemMetricsPublisherForV2.java | 1 + .../server/resourcemanager/rmapp/MockRMApp.java | 10 ++++ .../rmapp/TestRMAppTransitions.java | 2 + .../capacity/TestApplicationPriority.java | 13 +++--- 22 files changed, 181 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 4f952b7..3d7b2b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -616,7 +616,7 @@ public class ApplicationMasterService extends AbstractService implements // Set application priority allocateResponse.setApplicationPriority(app - .getApplicationSubmissionContext().getPriority()); + .getApplicationPriority()); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index c8af526..4e36b6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1602,14 +1602,14 @@ public class ClientRMService extends AbstractService implements .newRecordInstance(UpdateApplicationPriorityResponse.class); // Update priority only when app is tracked by the scheduler if (!ACTIVE_APP_STATES.contains(application.getState())) { - if (COMPLETED_APP_STATES.contains(application.getState())) { + if (application.isAppInCompletedStates()) { // If Application is in any of the final states, change priority // can be skipped rather throwing exception. RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId); response.setApplicationPriority(application - .getApplicationSubmissionContext().getPriority()); + .getApplicationPriority()); return response; } String msg = "Application in " + application.getState() @@ -1622,8 +1622,7 @@ public class ClientRMService extends AbstractService implements } try { - rmContext.getScheduler().updateApplicationPriority(newAppPriority, - applicationId); + rmAppManager.updateApplicationPriority(applicationId, newAppPriority); } catch (YarnException ex) { RMAuditLogger.logFailure(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "UNKNOWN", "ClientRMService", @@ -1633,8 +1632,7 @@ public class ClientRMService extends AbstractService implements RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId); - response.setApplicationPriority(application - .getApplicationSubmissionContext().getPriority()); + response.setApplicationPriority(application.getApplicationPriority()); return response; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index e566243..b5f02fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -357,9 +357,9 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, // Verify and get the update application priority and set back to // submissionContext - Priority appPriority = rmContext.getScheduler() - .checkAndGetApplicationPriority(submissionContext.getPriority(), user, - submissionContext.getQueue(), applicationId); + Priority appPriority = scheduler.checkAndGetApplicationPriority( + submissionContext.getPriority(), user, submissionContext.getQueue(), + applicationId); submissionContext.setPriority(appPriority); UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); @@ -521,6 +521,10 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, throws YarnException { ApplicationId applicationId = app.getApplicationId(); synchronized (applicationId) { + if (app.isAppInCompletedStates()) { + return; + } + Map<ApplicationTimeoutType, Long> newExpireTime = RMServerUtils .validateISO8601AndConvertToLocalTimeEpoch(newTimeoutInISO8601Format); @@ -548,4 +552,43 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>, ((RMAppImpl) app).updateApplicationTimeout(newExpireTime); } } + + /** + * updateApplicationPriority will invoke scheduler api to update the + * new priority to RM and StateStore. + * @param applicationId Application Id + * @param newAppPriority proposed new application priority + * @throws YarnException Handle exceptions + */ + public void updateApplicationPriority(ApplicationId applicationId, + Priority newAppPriority) throws YarnException { + RMApp app = this.rmContext.getRMApps().get(applicationId); + + synchronized (applicationId) { + if (app.isAppInCompletedStates()) { + return; + } + + // Create a future object to capture exceptions from StateStore. + SettableFuture<Object> future = SettableFuture.create(); + + // Invoke scheduler api to update priority in scheduler and to + // State Store. + Priority appPriority = rmContext.getScheduler() + .updateApplicationPriority(newAppPriority, applicationId, future); + + if (app.getApplicationPriority().equals(appPriority)) { + return; + } + + Futures.get(future, YarnException.class); + + // update in-memory + ((RMAppImpl) app).setApplicationPriority(appPriority); + } + + // Update the changed application state to timeline server + rmContext.getSystemMetricsPublisher().appUpdated(app, + System.currentTimeMillis()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index aa3ef0a..73bb301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -88,7 +88,7 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, app.getApplicationSubmissionContext().getUnmanagedAM()); entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); entityInfo.put(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION, app.getAmNodeLabelExpression()); entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, @@ -164,7 +164,7 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, app.getQueue()); eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE); tEvent.setTimestamp(updatedTime); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 6eb6eb9..412d573 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -117,7 +117,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, app.getApplicationSubmissionContext().getUnmanagedAM()); entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); entity.getConfigs().put( ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION, app.getAmNodeLabelExpression()); @@ -272,7 +272,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, app.getQueue()); eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ApplicationMetricsConstants.UPDATED_EVENT_TYPE); tEvent.setTimestamp(currentTimeMillis); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index cd08743..b3a87a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -283,4 +284,17 @@ public interface RMApp extends EventHandler<RMAppEvent> { CallerContext getCallerContext(); Map<ApplicationTimeoutType, Long> getApplicationTimeouts(); + + /** + * Get priority of the application. + * @return priority + */ + Priority getApplicationPriority(); + + /** + * To verify whether app has reached in its completing/completed states. + * + * @return True/False to confirm whether app is in final states + */ + boolean isAppInCompletedStates(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index a376311..a5962f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -197,6 +198,8 @@ public class RMAppImpl implements RMApp, Recoverable { Object transitionTodo; + private Priority applicationPriority; + private static final StateMachineFactory<RMAppImpl, RMAppState, RMAppEventType, @@ -461,6 +464,10 @@ public class RMAppImpl implements RMApp, Recoverable { this.applicationType = applicationType; this.applicationTags = applicationTags; this.amReq = amReq; + if (submissionContext.getPriority() != null) { + this.applicationPriority = Priority + .newInstance(submissionContext.getPriority().getPriority()); + } int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); @@ -533,8 +540,6 @@ public class RMAppImpl implements RMApp, Recoverable { DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD; } } - - } /** @@ -777,7 +782,7 @@ public class RMAppImpl implements RMApp, Recoverable { createApplicationState(), diags, trackingUrl, this.startTime, this.finishTime, finishState, appUsageReport, origTrackingUrl, progress, this.applicationType, amrmToken, applicationTags, - this.submissionContext.getPriority()); + this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); @@ -1138,14 +1143,14 @@ public class RMAppImpl implements RMApp, Recoverable { // started or started but not yet saved. if (app.attempts.isEmpty()) { app.scheduler.handle(new AppAddedSchedulerEvent(app.user, - app.submissionContext, false)); + app.submissionContext, false, app.applicationPriority)); return RMAppState.SUBMITTED; } // Add application to scheduler synchronously to guarantee scheduler // knows applications before AM or NM re-registers. app.scheduler.handle(new AppAddedSchedulerEvent(app.user, - app.submissionContext, true)); + app.submissionContext, true, app.applicationPriority)); // recover attempts app.recoverAppAttempts(); @@ -1162,7 +1167,7 @@ public class RMAppImpl implements RMApp, Recoverable { @Override public void transition(RMAppImpl app, RMAppEvent event) { app.handler.handle(new AppAddedSchedulerEvent(app.user, - app.submissionContext, false)); + app.submissionContext, false, app.applicationPriority)); // send the ATS create Event app.sendATSCreateEvent(); } @@ -1619,7 +1624,16 @@ public class RMAppImpl implements RMApp, Recoverable { return appState == RMAppState.FAILED || appState == RMAppState.FINISHED || appState == RMAppState.KILLED; } - + + @Override + public boolean isAppInCompletedStates() { + RMAppState appState = getState(); + return appState == RMAppState.FINISHED || appState == RMAppState.FINISHING + || appState == RMAppState.FAILED || appState == RMAppState.KILLED + || appState == RMAppState.FINAL_SAVING + || appState == RMAppState.KILLING; + } + public RMAppState getRecoveredFinalState() { return this.recoveredFinalState; } @@ -2018,4 +2032,13 @@ public class RMAppImpl implements RMApp, Recoverable { this.writeLock.unlock(); } } + + @Override + public Priority getApplicationPriority() { + return applicationPriority; + } + + public void setApplicationPriority(Priority applicationPriority) { + this.applicationPriority = applicationPriority; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 235e659..94af4dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -779,10 +779,12 @@ public abstract class AbstractYarnScheduler } @Override - public void updateApplicationPriority(Priority newPriority, - ApplicationId applicationId) throws YarnException { + public Priority updateApplicationPriority(Priority newPriority, + ApplicationId applicationId, SettableFuture<Object> future) + throws YarnException { // Dummy Implementation till Application Priority changes are done in // specific scheduler. + return Priority.newInstance(0); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index c4f575f..62ab7f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -51,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import com.google.common.util.concurrent.SettableFuture; + /** * This interface is used by the components to talk to the * scheduler for allocating of resources, cleaning up resources. @@ -318,9 +320,14 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> { * @param newPriority Submitted Application priority. * * @param applicationId Application ID + * + * @param future Sets any type of exception happened from StateStore + * + * @return updated priority */ - public void updateApplicationPriority(Priority newPriority, - ApplicationId applicationId) throws YarnException; + public Priority updateApplicationPriority(Priority newPriority, + ApplicationId applicationId, SettableFuture<Object> future) + throws YarnException; /** * http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 65a08c6..aea2e4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -145,6 +145,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.SettableFuture; @LimitedPrivate("yarn") @Evolving @@ -2405,8 +2406,9 @@ public class CapacityScheduler extends } @Override - public void updateApplicationPriority(Priority newPriority, - ApplicationId applicationId) throws YarnException { + public Priority updateApplicationPriority(Priority newPriority, + ApplicationId applicationId, SettableFuture<Object> future) + throws YarnException { Priority appPriority = null; SchedulerApplication<FiCaSchedulerApp> application = applications .get(applicationId); @@ -2417,38 +2419,36 @@ public class CapacityScheduler extends } RMApp rmApp = rmContext.getRMApps().get(applicationId); + appPriority = checkAndGetApplicationPriority(newPriority, rmApp.getUser(), rmApp.getQueue(), applicationId); if (application.getPriority().equals(appPriority)) { - return; + future.set(null); + return appPriority; } - // Update new priority in Submission Context to keep track in HA + // Update new priority in Submission Context to update to StateStore. rmApp.getApplicationSubmissionContext().setPriority(appPriority); // Update to state store - ApplicationStateData appState = - ApplicationStateData.newInstance(rmApp.getSubmitTime(), - rmApp.getStartTime(), rmApp.getApplicationSubmissionContext(), - rmApp.getUser(), rmApp.getCallerContext()); + ApplicationStateData appState = ApplicationStateData.newInstance( + rmApp.getSubmitTime(), rmApp.getStartTime(), + rmApp.getApplicationSubmissionContext(), rmApp.getUser(), + rmApp.getCallerContext()); appState.setApplicationTimeouts(rmApp.getApplicationTimeouts()); rmContext.getStateStore().updateApplicationStateSynchronously(appState, - false, null); + false, future); // As we use iterator over a TreeSet for OrderingPolicy, once we change // priority then reinsert back to make order correct. LeafQueue queue = (LeafQueue) getQueue(rmApp.getQueue()); - queue.updateApplicationPriority(application, appPriority); - // Update the changed application state to timeline server - rmContext.getSystemMetricsPublisher().appUpdated(rmApp, - System.currentTimeMillis()); - LOG.info("Priority '" + appPriority + "' is updated in queue :" + rmApp.getQueue() + " for application: " + applicationId + " for the user: " + rmApp.getUser()); + return appPriority; } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index 89d2f66..0a8d6fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -43,10 +43,11 @@ public class AppAddedSchedulerEvent extends SchedulerEvent { } public AppAddedSchedulerEvent(String user, - ApplicationSubmissionContext submissionContext, boolean isAppRecovering) { + ApplicationSubmissionContext submissionContext, boolean isAppRecovering, + Priority appPriority) { this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), - submissionContext.getPriority()); + appPriority); } public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 2c61339..852f937 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -1248,7 +1248,7 @@ public class RMWebServices extends WebServices { AppPriority ret = new AppPriority(); ret.setPriority( - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); return ret; } @@ -1289,7 +1289,7 @@ public class RMWebServices extends WebServices { "Trying to update priority an absent application " + appId); throw e; } - Priority priority = app.getApplicationSubmissionContext().getPriority(); + Priority priority = app.getApplicationPriority(); if (priority == null || priority.getPriority() != targetPriority.getPriority()) { return modifyApplicationPriority(app, callerUGI, @@ -1336,7 +1336,7 @@ public class RMWebServices extends WebServices { } } AppPriority ret = new AppPriority( - app.getApplicationSubmissionContext().getPriority().getPriority()); + app.getApplicationPriority().getPriority()); return Response.status(Status.OK).entity(ret).build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 42449b0..d3ddf80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -148,10 +148,9 @@ public class AppInfo { this.name = app.getName().toString(); this.queue = app.getQueue().toString(); this.priority = 0; - ApplicationSubmissionContext appSubmissionContext = - app.getApplicationSubmissionContext(); - if (appSubmissionContext.getPriority() != null) { - this.priority = appSubmissionContext.getPriority() + + if (app.getApplicationPriority() != null) { + this.priority = app.getApplicationPriority() .getPriority(); } this.progress = app.getProgress() * 100; @@ -220,6 +219,8 @@ public class AppInfo { vcoreSeconds = appMetrics.getVcoreSeconds(); preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds(); preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds(); + ApplicationSubmissionContext appSubmissionContext = + app.getApplicationSubmissionContext(); unmanagedApplication = appSubmissionContext.getUnmanagedAM(); appNodeLabelExpression = http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 981b045..03cc3a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -251,6 +251,7 @@ public class TestAppManager{ asContext.setApplicationId(appId); asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory)); asContext.setResource(mockResource()); + asContext.setPriority(Priority.newInstance(0)); setupDispatcher(rmContext, conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index 93befcb..00466ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -461,12 +461,10 @@ public class TestApplicationMasterService { AllocateResponse response1 = am1.allocate(allocateRequest); Assert.assertEquals(appPriority1, response1.getApplicationPriority()); - // get scheduler - CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); - // Change the priority of App1 to 8 Priority appPriority2 = Priority.newInstance(8); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + rm.getRMAppManager().updateApplicationPriority(app1.getApplicationId(), + appPriority2); AllocateResponse response2 = am1.allocate(allocateRequest); Assert.assertEquals(appPriority2, response2.getApplicationPriority()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index e48d9d2..f35efa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -456,7 +456,7 @@ public class TestClientRMService { } } - public ClientRMService createRMService() throws IOException { + public ClientRMService createRMService() throws IOException, YarnException { YarnScheduler yarnScheduler = mockYarnScheduler(); RMContext rmContext = mock(RMContext.class); mockRMContext(yarnScheduler, rmContext); @@ -968,6 +968,7 @@ public class TestClientRMService { submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); + submissionContext.setPriority(Priority.newInstance(0)); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); @@ -1042,6 +1043,7 @@ public class TestClientRMService { ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); when(asContext.getMaxAppAttempts()).thenReturn(1); when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression); + when(asContext.getPriority()).thenReturn(Priority.newInstance(0)); RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, @@ -1076,6 +1078,7 @@ public class TestClientRMService { attempts.put(attemptId, rmAppAttemptImpl); when(app.getCurrentAppAttempt()).thenReturn(rmAppAttemptImpl); when(app.getAppAttempts()).thenReturn(attempts); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(0)); when(rmAppAttemptImpl.getMasterContainer()).thenReturn(container); ResourceScheduler rs = mock(ResourceScheduler.class); when(rmContext.getScheduler()).thenReturn(rs); @@ -1098,7 +1101,7 @@ public class TestClientRMService { return app; } - private static YarnScheduler mockYarnScheduler() { + private static YarnScheduler mockYarnScheduler() throws YarnException { YarnScheduler yarnScheduler = mock(YarnScheduler.class); when(yarnScheduler.getMinimumResourceCapability()).thenReturn( Resources.createResource( @@ -1116,6 +1119,9 @@ public class TestClientRMService { ResourceCalculator rs = mock(ResourceCalculator.class); when(yarnScheduler.getResourceCalculator()).thenReturn(rs); + when(yarnScheduler.checkAndGetApplicationPriority(any(Priority.class), + anyString(), anyString(), any(ApplicationId.class))) + .thenReturn(Priority.newInstance(0)); return yarnScheduler; } @@ -1675,8 +1681,7 @@ public class TestClientRMService { RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority)); Assert.assertEquals("Incorrect priority has been set to application", - appPriority, app1.getApplicationSubmissionContext().getPriority() - .getPriority()); + appPriority, app1.getApplicationPriority().getPriority()); appPriority = 11; ClientRMService rmService = rm.getClientRMService(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index f21c7c0..9be52c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -235,6 +236,16 @@ public abstract class MockAsm extends MockApps { public Map<ApplicationTimeoutType, Long> getApplicationTimeouts() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public Priority getApplicationPriority() { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public boolean isAppInCompletedStates() { + throw new UnsupportedOperationException("Not supported yet."); + } } public static RMApp newApplication(int i) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 2b1740f..55e93c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -126,6 +126,7 @@ public class TestSystemMetricsPublisher { .thenReturn(Collections.singletonList("java -Xmx1024m")); when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext); when(app.getApplicationSubmissionContext()).thenReturn(asc); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(1)); metricsPublisher.appUpdated(app, 4L); } else { metricsPublisher.appUpdated(app, 4L); @@ -527,6 +528,7 @@ public class TestSystemMetricsPublisher { when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); when(app.getAMResourceRequest()).thenReturn(amReq); when(app.getAmNodeLabelExpression()).thenCallRealMethod(); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); when(app.getCallerContext()) .thenReturn(new CallerContext.Builder("context").build()); return app; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 3ab2501..2d40c91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -362,6 +362,7 @@ public class TestSystemMetricsPublisherForV2 { when(appSubmissionContext.getPriority()) .thenReturn(Priority.newInstance(0)); + when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); ContainerLaunchContext containerLaunchContext = mock(ContainerLaunchContext.class); when(containerLaunchContext.getCommands()) http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index bbfa60f..48a60b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -321,4 +321,14 @@ public class MockRMApp implements RMApp { public Map<ApplicationTimeoutType, Long> getApplicationTimeouts() { throw new UnsupportedOperationException("Not supported yet."); } + + @Override + public Priority getApplicationPriority() { + return null; + } + + @Override + public boolean isAppInCompletedStates() { + return false; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 6bf5ecf..c4459ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -266,6 +267,7 @@ public class TestRMAppTransitions { // applicationId will not be used because RMStateStore is mocked, // but applicationId is still set for safety submissionContext.setApplicationId(applicationId); + submissionContext.setPriority(Priority.newInstance(0)); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, http://git-wip-us.apache.org/repos/asf/hadoop/blob/a926f895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index 2a346f8..164ca20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -344,7 +344,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 8 Priority appPriority2 = Priority.newInstance(8); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null); // get scheduler app FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications() @@ -378,7 +378,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 15 Priority appPriority2 = Priority.newInstance(15); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null); // get scheduler app FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications() @@ -428,7 +428,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 8 Priority appPriority2 = Priority.newInstance(8); - cs.updateApplicationPriority(appPriority2, app1.getApplicationId()); + cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null); // let things settle down Thread.sleep(1000); @@ -449,8 +449,7 @@ public class TestApplicationPriority { .get(app1.getApplicationId()); // Verify whether priority 15 is reset to 10 - Assert.assertEquals(appPriority2, loadedApp.getCurrentAppAttempt() - .getSubmissionContext().getPriority()); + Assert.assertEquals(appPriority2, loadedApp.getApplicationPriority()); rm2.stop(); rm1.stop(); @@ -558,7 +557,7 @@ public class TestApplicationPriority { // Change the priority of App1 to 3 (lowest) Priority appPriority3 = Priority.newInstance(3); - cs.updateApplicationPriority(appPriority3, app2.getApplicationId()); + cs.updateApplicationPriority(appPriority3, app2.getApplicationId(), null); // add request for containers App2 am2.allocate("127.0.0.1", 2 * GB, 3, new ArrayList<ContainerId>()); @@ -790,7 +789,7 @@ public class TestApplicationPriority { throws YarnException { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); cs.updateApplicationPriority(Priority.newInstance(2), - app.getApplicationId()); + app.getApplicationId(), null); SchedulerEvent removeAttempt; removeAttempt = new AppAttemptRemovedSchedulerEvent( app.getCurrentAppAttempt().getAppAttemptId(), RMAppAttemptState.KILLED, --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
