YARN-5965. Retrospect ApplicationReport#getApplicationTimeouts. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab923a53 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab923a53 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab923a53 Branch: refs/heads/YARN-5972 Commit: ab923a53fcf55d4d75aa027d46e3c4a659015325 Parents: c73e08a Author: Sunil <sun...@apache.org> Authored: Thu Dec 8 00:27:25 2016 +0530 Committer: Sunil <sun...@apache.org> Committed: Thu Dec 8 00:27:25 2016 +0530 ---------------------------------------------------------------------- .../yarn/api/records/ApplicationReport.java | 6 +- .../src/main/proto/yarn_protos.proto | 7 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 3 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 7 +- .../impl/pb/ApplicationReportPBImpl.java | 107 +++++++++++-------- .../server/resourcemanager/rmapp/RMAppImpl.java | 13 ++- .../rmapp/TestApplicationLifetimeMonitor.java | 15 ++- 7 files changed, 91 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index c039514..9e9ec3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; -import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -451,10 +451,10 @@ public abstract class ApplicationReport { @Public @Unstable - public abstract List<ApplicationTimeout> getApplicationTimeouts(); + public abstract Map<ApplicationTimeoutType, ApplicationTimeout> getApplicationTimeouts(); @Private @Unstable public abstract void setApplicationTimeouts( - List<ApplicationTimeout> timeouts); + Map<ApplicationTimeoutType, ApplicationTimeout> timeouts); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 43a661f..5a70298 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -214,7 +214,12 @@ message ApplicationReportProto { optional PriorityProto priority = 23; optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; - repeated ApplicationTimeoutProto application_timeouts = 26; + repeated AppTimeoutsMapProto appTimeouts = 26; +} + +message AppTimeoutsMapProto { + optional ApplicationTimeoutTypeProto application_timeout_type = 1; + optional ApplicationTimeoutProto application_timeout = 2; } message ApplicationTimeoutProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 20a65bf..efe5921 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -714,7 +714,8 @@ public class ApplicationCLI extends YarnCLI { appReportStr.println(appReport.getAppNodeLabelExpression()); appReportStr.print("\tAM container Node Label Expression : "); appReportStr.println(appReport.getAmNodeLabelExpression()); - for (ApplicationTimeout timeout : appReport.getApplicationTimeouts()) { + for (ApplicationTimeout timeout : appReport.getApplicationTimeouts() + .values()) { appReportStr.print("\tTimeoutType : " + timeout.getTimeoutType()); appReportStr.print("\tExpiryTime : " + timeout.getExpiryTime()); appReportStr.println( http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index f9ec5c7..a677606 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -130,8 +130,8 @@ public class TestYarnCLI { newApplicationReport.setPriority(Priority.newInstance(0)); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", -1); - newApplicationReport - .setApplicationTimeouts(Collections.singletonList(timeout)); + newApplicationReport.setApplicationTimeouts( + Collections.singletonMap(timeout.getTimeoutType(), timeout)); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -2104,7 +2104,8 @@ public class TestYarnCLI { "N/A", 0.53789f, "YARN", null); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1); - appReport.setApplicationTimeouts(Collections.singletonList(timeout)); + appReport.setApplicationTimeouts( + Collections.singletonMap(timeout.getTimeoutType(), timeout)); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(appReport); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index f4987d3..036fa90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -26,11 +26,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.proto.YarnProtos.AppTimeoutsMapProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; @@ -44,10 +46,11 @@ import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.TextFormat; -import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; @Private @@ -63,7 +66,7 @@ public class ApplicationReportPBImpl extends ApplicationReport { private Token amRmToken = null; private Set<String> applicationTags = null; private Priority priority = null; - private List<ApplicationTimeout> applicationTimeoutList = null; + private Map<ApplicationTimeoutType, ApplicationTimeout> applicationTimeouts = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -498,8 +501,8 @@ public class ApplicationReportPBImpl extends ApplicationReport { builder.getPriority())) { builder.setPriority(convertToProtoFormat(this.priority)); } - if (this.applicationTimeoutList != null) { - addLocalApplicationTimeoutToProto(); + if (this.applicationTimeouts != null) { + addApplicationTimeouts(); } } @@ -679,76 +682,86 @@ public class ApplicationReportPBImpl extends ApplicationReport { } @Override - public List<ApplicationTimeout> getApplicationTimeouts() { - initLocalApplicationsList(); - return this.applicationTimeoutList; + public Map<ApplicationTimeoutType, ApplicationTimeout> getApplicationTimeouts() { + initApplicationTimeout(); + return this.applicationTimeouts; } - private void initLocalApplicationsList() { - if (this.applicationTimeoutList != null) { + @Override + public void setApplicationTimeouts( + Map<ApplicationTimeoutType, ApplicationTimeout> timeouts) { + if (timeouts == null) { return; } - ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; - List<ApplicationTimeoutProto> list = p.getApplicationTimeoutsList(); - this.applicationTimeoutList = new ArrayList<ApplicationTimeout>(); + initApplicationTimeout(); + this.applicationTimeouts.clear(); + this.applicationTimeouts.putAll(timeouts); + } - for (ApplicationTimeoutProto a : list) { - this.applicationTimeoutList.add(convertFromProtoFormat(a)); + private void initApplicationTimeout() { + if (this.applicationTimeouts != null) { + return; + } + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + List<AppTimeoutsMapProto> lists = p.getAppTimeoutsList(); + this.applicationTimeouts = + new HashMap<ApplicationTimeoutType, ApplicationTimeout>(lists.size()); + for (AppTimeoutsMapProto timeoutProto : lists) { + this.applicationTimeouts.put( + ProtoUtils + .convertFromProtoFormat(timeoutProto.getApplicationTimeoutType()), + convertFromProtoFormat(timeoutProto.getApplicationTimeout())); } } - private void addLocalApplicationTimeoutToProto() { + private ApplicationTimeoutPBImpl convertFromProtoFormat( + ApplicationTimeoutProto p) { + return new ApplicationTimeoutPBImpl(p); + } + + private ApplicationTimeoutProto convertToProtoFormat(ApplicationTimeout t) { + return ((ApplicationTimeoutPBImpl) t).getProto(); + } + + private void addApplicationTimeouts() { maybeInitBuilder(); - builder.clearApplicationTimeouts(); - if (applicationTimeoutList == null) { + builder.clearAppTimeouts(); + if (applicationTimeouts == null) { return; } - Iterable<ApplicationTimeoutProto> iterable = - new Iterable<ApplicationTimeoutProto>() { - @Override - public Iterator<ApplicationTimeoutProto> iterator() { - return new Iterator<ApplicationTimeoutProto>() { + Iterable<? extends AppTimeoutsMapProto> values = + new Iterable<AppTimeoutsMapProto>() { - private Iterator<ApplicationTimeout> iter = - applicationTimeoutList.iterator(); + @Override + public Iterator<AppTimeoutsMapProto> iterator() { + return new Iterator<AppTimeoutsMapProto>() { + private Iterator<ApplicationTimeoutType> iterator = + applicationTimeouts.keySet().iterator(); @Override public boolean hasNext() { - return iter.hasNext(); + return iterator.hasNext(); } @Override - public ApplicationTimeoutProto next() { - return convertToProtoFormat(iter.next()); + public AppTimeoutsMapProto next() { + ApplicationTimeoutType key = iterator.next(); + return AppTimeoutsMapProto.newBuilder() + .setApplicationTimeout( + convertToProtoFormat(applicationTimeouts.get(key))) + .setApplicationTimeoutType( + ProtoUtils.convertToProtoFormat(key)) + .build(); } @Override public void remove() { throw new UnsupportedOperationException(); - } }; - } }; - builder.addAllApplicationTimeouts(iterable); + this.builder.addAllAppTimeouts(values); } - private ApplicationTimeoutPBImpl convertFromProtoFormat( - ApplicationTimeoutProto p) { - return new ApplicationTimeoutPBImpl(p); - } - - private ApplicationTimeoutProto convertToProtoFormat(ApplicationTimeout t) { - return ((ApplicationTimeoutPBImpl) t).getProto(); - } - - @Override - public void setApplicationTimeouts(List<ApplicationTimeout> timeouts) { - maybeInitBuilder(); - if (timeouts == null) { - builder.clearApplicationTimeouts(); - } - this.applicationTimeoutList = timeouts; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 1f1586a..a647969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -790,10 +790,17 @@ public class RMAppImpl implements RMApp, Recoverable { long timeoutInMillis = applicationTimeouts .get(ApplicationTimeoutType.LIFETIME).longValue(); timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); - timeout.setRemainingTime( - Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0)); + if (isAppInCompletedStates()) { + // if application configured with timeout and finished before timeout + // happens then remaining time should not be calculated. + timeout.setRemainingTime(0); + } else { + timeout.setRemainingTime( + Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0)); + } } - report.setApplicationTimeouts(Collections.singletonList(timeout)); + report.setApplicationTimeouts( + Collections.singletonMap(timeout.getTimeoutType(), timeout)); return report; } finally { this.readLock.unlock(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab923a53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index aa4acca..fdc47b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -24,7 +24,6 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; @@ -136,15 +135,13 @@ public class TestApplicationLifetimeMonitor { GetApplicationReportRequest appRequest = recordFactory.newRecordInstance(GetApplicationReportRequest.class); appRequest.setApplicationId(app2.getApplicationId()); - List<ApplicationTimeout> applicationTimeoutList = rm.getRMContext() - .getClientRMService().getApplicationReport(appRequest) + Map<ApplicationTimeoutType, ApplicationTimeout> appTimeouts = rm + .getRMContext().getClientRMService().getApplicationReport(appRequest) .getApplicationReport().getApplicationTimeouts(); - Assert.assertTrue("Application Timeout list are empty.", - !applicationTimeoutList.isEmpty()); - ApplicationTimeout timeout = applicationTimeoutList.iterator().next(); - Assert.assertEquals("Application timeout Type is incorrect.", - ApplicationTimeoutType.LIFETIME.toString(), - timeout.getTimeoutType().toString()); + Assert.assertTrue("Application Timeout are empty.", + !appTimeouts.isEmpty()); + ApplicationTimeout timeout = + appTimeouts.get(ApplicationTimeoutType.LIFETIME); Assert.assertEquals("Application timeout string is incorrect.", formatISO8601, timeout.getExpiryTime()); Assert.assertTrue("Application remaining time is incorrect", --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org