YARN-5384. Expose priority in ReservationSystem submission APIs. (Sean Po via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a3697de Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a3697de Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a3697de Branch: refs/heads/YARN-2915 Commit: 3a3697deab3e3397082222deb66fb613d86ff9ae Parents: 89bd6d2 Author: Subru Krishnan <su...@apache.org> Authored: Fri Sep 30 19:41:43 2016 -0700 Committer: Subru Krishnan <su...@apache.org> Committed: Fri Sep 30 19:41:43 2016 -0700 ---------------------------------------------------------------------- .../yarn/api/records/ReservationDefinition.java | 44 ++++++++++++++++---- .../src/main/proto/yarn_protos.proto | 1 + .../impl/pb/ReservationDefinitionPBImpl.java | 31 ++++++++++++++ .../webapp/dao/ReservationDefinitionInfo.java | 11 +++++ .../reservation/ReservationSystemTestUtil.java | 10 ++++- .../src/site/markdown/ResourceManagerRest.md | 3 ++ 6 files changed, 91 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3697de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java index 8ef881b..bb9bca2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.util.Records; @@ -38,7 +37,7 @@ public abstract class ReservationDefinition { @Unstable public static ReservationDefinition newInstance(long arrival, long deadline, ReservationRequests reservationRequests, String name, - String recurrenceExpression) { + String recurrenceExpression, Priority priority) { ReservationDefinition rDefinition = Records.newRecord(ReservationDefinition.class); rDefinition.setArrival(arrival); @@ -46,6 +45,7 @@ public abstract class ReservationDefinition { rDefinition.setReservationRequests(reservationRequests); rDefinition.setReservationName(name); rDefinition.setRecurrenceExpression(recurrenceExpression); + rDefinition.setPriority(priority); return rDefinition; } @@ -53,8 +53,8 @@ public abstract class ReservationDefinition { @Unstable public static ReservationDefinition newInstance(long arrival, long deadline, ReservationRequests reservationRequests, String name) { - ReservationDefinition rDefinition = - newInstance(arrival, deadline, reservationRequests, name, "0"); + ReservationDefinition rDefinition = newInstance(arrival, deadline, + reservationRequests, name, "0", Priority.UNDEFINED); return rDefinition; } @@ -130,7 +130,7 @@ public abstract class ReservationDefinition { * allocation in the scheduler */ @Public - @Evolving + @Unstable public abstract String getReservationName(); /** @@ -142,7 +142,7 @@ public abstract class ReservationDefinition { * allocation in the scheduler */ @Public - @Evolving + @Unstable public abstract void setReservationName(String name); /** @@ -160,7 +160,7 @@ public abstract class ReservationDefinition { * @return recurrence of this reservation */ @Public - @Evolving + @Unstable public abstract String getRecurrenceExpression(); /** @@ -178,7 +178,35 @@ public abstract class ReservationDefinition { * @param recurrenceExpression recurrence interval of this reservation */ @Public - @Evolving + @Unstable public abstract void setRecurrenceExpression(String recurrenceExpression); + /** + * Get the priority for this reservation. A lower number for priority + * indicates a higher priority reservation. Recurring reservations are + * always higher priority than non-recurring reservations. Priority for + * non-recurring reservations are only compared with non-recurring + * reservations. Likewise for recurring reservations. + * + * @return int representing the priority of the reserved resource + * allocation in the scheduler + */ + @Public + @Unstable + public abstract Priority getPriority(); + + /** + * Set the priority for this reservation. A lower number for priority + * indicates a higher priority reservation. Recurring reservations are + * always higher priority than non-recurring reservations. Priority for + * non-recurring reservations are only compared with non-recurring + * reservations. Likewise for recurring reservations. + * + * @param priority representing the priority of the reserved resource + * allocation in the scheduler + */ + @Public + @Unstable + public abstract void setPriority(Priority priority); + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3697de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index f788295..9c746fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -489,6 +489,7 @@ message ReservationDefinitionProto { optional int64 deadline = 3; optional string reservation_name = 4; optional string recurrence_expression = 5 [default = "0"]; + optional PriorityProto priority = 6; } message ResourceAllocationRequestProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3697de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java index b30cd2a..49aef11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationRequests; +import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProto; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationDefinitionProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestsProto; @@ -32,6 +34,7 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition { boolean viaProto = false; private ReservationRequests reservationReqs; + private Priority priority = null; public ReservationDefinitionPBImpl() { builder = ReservationDefinitionProto.newBuilder(); @@ -150,6 +153,33 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition { builder.setReservationName(name); } + @Override + public Priority getPriority() { + ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder; + if (this.priority != null) { + return this.priority; + } + if (!p.hasPriority()) { + return Priority.UNDEFINED; + } + this.priority = convertFromProtoFormat(p.getPriority()); + return this.priority; + } + + @Override + public void setPriority(Priority priority) { + maybeInitBuilder(); + if (priority == null) { + this.priority = Priority.UNDEFINED; + } + this.priority = priority; + } + + private PriorityPBImpl convertFromProtoFormat( + YarnProtos.PriorityProto p) { + return new PriorityPBImpl(p); + } + private ReservationRequestsPBImpl convertFromProtoFormat( ReservationRequestsProto p) { return new ReservationRequestsPBImpl(p); @@ -164,6 +194,7 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition { return "{Arrival: " + getArrival() + ", Deadline: " + getDeadline() + ", Reservation Name: " + getReservationName() + ", Recurrence expression: " + getRecurrenceExpression() + + ", Priority: " + getPriority().toString() + ", Resources: " + getReservationRequests() + "}"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3697de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java index 71ee924..42a07af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ReservationDefinitionInfo.java @@ -44,6 +44,9 @@ public class ReservationDefinitionInfo { @XmlElement(name = "reservation-name") private String reservationName; + @XmlElement(name = "priority") + private int priority; + public ReservationDefinitionInfo() { } @@ -89,4 +92,12 @@ public class ReservationDefinitionInfo { this.reservationName = reservationName; } + public int getPriority() { + return priority; + } + + public void setPriority(int priority) { + this.priority = priority; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3697de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java index 24c386a..1ff6a1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java @@ -31,6 +31,7 @@ import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationRequest; @@ -199,6 +200,13 @@ public class ReservationSystemTestUtil { public static ReservationSubmissionRequest createSimpleReservationRequest( ReservationId reservationId, int numContainers, long arrival, long deadline, long duration) { + return createSimpleReservationRequest(reservationId, numContainers, + arrival, deadline, duration, Priority.UNDEFINED); + } + + public static ReservationSubmissionRequest createSimpleReservationRequest( + ReservationId reservationId, int numContainers, long arrival, + long deadline, long duration, Priority priority) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), @@ -208,7 +216,7 @@ public class ReservationSystemTestUtil { ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, - "testClientRMService#reservation"); + "testClientRMService#reservation", "0", priority); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, reservationQ, reservationId); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a3697de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index 5862506..051509c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -3237,6 +3237,7 @@ The Cluster Reservation API can be used to list reservations. When listing reser | deadline | long | The UTC time representation of the latest time within which this reservation can be allocated. | | reservation-name | string | A mnemonic name of the reservation (not a valid identifier). | | reservation-requests | object | A list of "stages" or phases of this reservation, each describing resource requirements and duration | +| priority | int | An integer representing the priority of the reservation. A lower number for priority indicates a higher priority reservation. Recurring reservations are always higher priority than non-recurring reservations. Priority for non-recurring reservations are only compared with non-recurring reservations. Likewise with recurring reservations. | ### Elements of the *reservation-requests* object @@ -3500,6 +3501,7 @@ Elements of the *reservation-definition* object | deadline | long | The UTC time representation of the latest time within which this reservation can be allocated. | | reservation-name | string | A mnemonic name of the reservation (not a valid identifier). | | reservation-requests | object | A list of "stages" or phases of this reservation, each describing resource requirements and duration | +| priority | int | An integer representing the priority of the reservation. A lower number for priority indicates a higher priority reservation. Recurring reservations are always higher priority than non-recurring reservations. Priority for non-recurring reservations are only compared with non-recurring reservations. Likewise with recurring reservations. | Elements of the *reservation-requests* object @@ -3675,6 +3677,7 @@ Elements of the *reservation-definition* object | deadline | long | The UTC time representation of the latest time within which this reservation can be allocated. | | reservation-name | string | A mnemonic name of the reservation (not a valid identifier). | | reservation-requests | object | A list of "stages" or phases of this reservation, each describing resource requirements and duration | +| priority | int | An integer representing the priority of the reservation. A lower number for priority indicates a higher priority reservation. Recurring reservations are always higher priority than non-recurring reservations. Priority for non-recurring reservations are only compared with non-recurring reservations. Likewise with recurring reservations. | Elements of the *reservation-requests* object --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
