YARN-6819. Application report fails if app rejected due to nodesize. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/845c4e52 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/845c4e52 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/845c4e52 Branch: refs/heads/HDFS-7240 Commit: 845c4e52bdd579a24df5dbba7477b0ebf2fa16f1 Parents: daaf530 Author: Rohith Sharma K S <rohithsharm...@apache.org> Authored: Wed Jul 19 11:10:52 2017 +0530 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Wed Jul 19 11:10:52 2017 +0530 ---------------------------------------------------------------------- .../resourcemanager/recovery/RMStateStore.java | 5 ++-- .../resourcemanager/rmapp/RMAppEvent.java | 24 -------------------- .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../server/resourcemanager/rmapp/RMAppImpl.java | 8 +++---- .../recovery/TestZKRMStateStore.java | 14 +++++++----- .../rmapp/TestRMAppTransitions.java | 17 ++++++++++++++ 6 files changed, 33 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/845c4e52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index e945b59..d0a8cf5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -221,8 +221,9 @@ public abstract class RMStateStore extends AbstractService { } catch (Exception e) { LOG.error("Error storing app: " + appId, e); if (e instanceof StoreLimitException) { - store.notifyApplication(new RMAppEvent(appId, - RMAppEventType.APP_REJECTED, e.getMessage(), false)); + store.notifyApplication( + new RMAppEvent(appId, RMAppEventType.APP_SAVE_FAILED, + e.getMessage())); } else { isFenced = store.notifyStoreOperationFailedInternal(e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/845c4e52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java index 0c6139e..5c46945 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java @@ -25,7 +25,6 @@ public class RMAppEvent extends AbstractEvent<RMAppEventType>{ private final ApplicationId appId; private final String diagnosticMsg; - private boolean storeAppInfo; public RMAppEvent(ApplicationId appId, RMAppEventType type) { this(appId, type, ""); @@ -36,21 +35,6 @@ public class RMAppEvent extends AbstractEvent<RMAppEventType>{ super(type); this.appId = appId; this.diagnosticMsg = diagnostic; - this.storeAppInfo = true; - } - - /** - * Constructor to create RM Application Event type. - * - * @param appId application Id - * @param type RM Event type - * @param diagnostic Diagnostic message for event - * @param storeApp Application should be saved or not - */ - public RMAppEvent(ApplicationId appId, RMAppEventType type, String diagnostic, - boolean storeApp) { - this(appId, type, diagnostic); - this.storeAppInfo = storeApp; } public ApplicationId getApplicationId() { @@ -61,12 +45,4 @@ public class RMAppEvent extends AbstractEvent<RMAppEventType>{ return this.diagnosticMsg; } - /** - * Store application to state store or not. - * - * @return boolean application should be saved to store. - */ - public boolean doStoreAppInfo() { - return storeAppInfo; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/845c4e52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index aa5d6f0..04d2db5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -47,4 +47,5 @@ public enum RMAppEventType { // Source: RMStateStore APP_NEW_SAVED, APP_UPDATE_SAVED, + APP_SAVE_FAILED, } http://git-wip-us.apache.org/repos/asf/hadoop/blob/845c4e52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index bf8fa4f..fa2f20c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -243,6 +243,8 @@ public class RMAppImpl implements RMApp, Recoverable { RMAppEventType.APP_REJECTED, new FinalSavingTransition(new AppRejectedTransition(), RMAppState.FAILED)) + .addTransition(RMAppState.NEW_SAVING, RMAppState.FAILED, + RMAppEventType.APP_SAVE_FAILED, new AppRejectedTransition()) // Transitions from SUBMITTED state .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, @@ -1307,10 +1309,8 @@ public class RMAppImpl implements RMApp, Recoverable { @Override public void transition(RMAppImpl app, RMAppEvent event) { - if (event.doStoreAppInfo()) { - app.rememberTargetTransitionsAndStoreState(event, transitionToDo, - targetedFinalState, stateToBeStored); - } + app.rememberTargetTransitionsAndStoreState(event, transitionToDo, + targetedFinalState, stateToBeStored); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/845c4e52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index fcd8647..942e9e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -270,20 +270,22 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { - return dispatcher.apprejectedEvnt; + return dispatcher.appsavefailedEvnt; } }, 100, 5000); } static class TestAppRejDispatcher extends TestDispatcher { - private boolean apprejectedEvnt; + private boolean appsavefailedEvnt; public void handle(Event event) { - if (event instanceof RMAppEvent - && event.getType().equals(RMAppEventType.APP_REJECTED)) { - apprejectedEvnt = true; + if (event instanceof RMAppEvent && event.getType() + .equals(RMAppEventType.APP_SAVE_FAILED)) { + appsavefailedEvnt = true; } - }; + } + + ; } @Test (timeout = 60000) http://git-wip-us.apache.org/repos/asf/hadoop/blob/845c4e52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 5aa7af9..7c54b60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -653,6 +653,23 @@ public class TestRMAppTransitions { } @Test (timeout = 30000) + public void testAppNewSavingSaveReject() throws IOException { + LOG.info("--- START: testAppNewSavingSaveReject ---"); + RMApp application = testCreateAppNewSaving(null); + // NEW_SAVING => FAILED event RMAppEventType.APP_SAVE_FAILED + String rejectedText = "Test Application Rejected"; + RMAppEvent event = new RMAppEvent(application.getApplicationId(), + RMAppEventType.APP_SAVE_FAILED, rejectedText); + application.handle(event); + rmDispatcher.await(); + assertFailed(application, rejectedText); + verify(store, times(0)).updateApplicationState( + any(ApplicationStateData.class)); + verifyApplicationFinished(RMAppState.FAILED); + assertTimesAtFinish(application); + } + + @Test (timeout = 30000) public void testAppSubmittedRejected() throws IOException { LOG.info("--- START: testAppSubmittedRejected ---"); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org