YARN-5113. Refactoring and other clean-up for distributed scheduling. (Konstantinos Karanasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5766b1d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5766b1d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5766b1d Branch: refs/heads/HADOOP-12756 Commit: e5766b1dbee02ae0ef89618e172f3fb227af19e8 Parents: 3d191cc Author: Arun Suresh <asur...@apache.org> Authored: Sun Jul 31 10:18:01 2016 -0700 Committer: Arun Suresh <asur...@apache.org> Committed: Sun Jul 31 11:48:25 2016 -0700 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 93 ++-- .../yarn/conf/TestYarnConfigurationFields.java | 35 -- .../api/impl/TestDistributedScheduling.java | 6 +- .../src/main/resources/yarn-default.xml | 146 ++++++- .../hadoop-yarn-server-common/pom.xml | 2 +- .../api/DistributedSchedulerProtocol.java | 78 ---- .../api/DistributedSchedulerProtocolPB.java | 36 -- .../api/DistributedSchedulingAMProtocol.java | 87 ++++ .../api/DistributedSchedulingAMProtocolPB.java | 37 ++ .../hadoop/yarn/server/api/ServerRMProxy.java | 2 +- ...istributedSchedulerProtocolPBClientImpl.java | 154 ------- ...ributedSchedulingAMProtocolPBClientImpl.java | 153 +++++++ ...stributedSchedulerProtocolPBServiceImpl.java | 146 ------- ...ibutedSchedulingAMProtocolPBServiceImpl.java | 151 +++++++ .../DistSchedAllocateRequest.java | 69 --- .../DistSchedAllocateResponse.java | 58 --- .../DistSchedRegisterResponse.java | 102 ----- .../DistributedSchedulingAllocateRequest.java | 71 +++ .../DistributedSchedulingAllocateResponse.java | 66 +++ ...RegisterDistributedSchedulingAMResponse.java | 108 +++++ .../impl/pb/DistSchedAllocateRequestPBImpl.java | 185 -------- .../pb/DistSchedAllocateResponsePBImpl.java | 180 -------- .../pb/DistSchedRegisterResponsePBImpl.java | 308 ------------- ...tributedSchedulingAllocateRequestPBImpl.java | 187 ++++++++ ...ributedSchedulingAllocateResponsePBImpl.java | 190 ++++++++ ...erDistributedSchedulingAMResponsePBImpl.java | 332 ++++++++++++++ .../proto/distributed_scheduler_protocol.proto | 39 -- .../distributed_scheduling_am_protocol.proto | 39 ++ .../yarn_server_common_service_protos.proto | 12 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 7 +- .../amrmproxy/AbstractRequestInterceptor.java | 20 +- .../amrmproxy/DefaultRequestInterceptor.java | 41 +- .../amrmproxy/RequestInterceptor.java | 4 +- .../scheduler/DistributedScheduler.java | 426 ++++++++++++++++++ .../nodemanager/scheduler/LocalScheduler.java | 432 ------------------- .../OpportunisticContainerAllocator.java | 66 +-- .../scheduler/TestDistributedScheduler.java | 270 ++++++++++++ .../scheduler/TestLocalScheduler.java | 270 ------------ .../DistributedSchedulingAMService.java | 361 ++++++++++++++++ .../DistributedSchedulingService.java | 358 --------------- .../server/resourcemanager/ResourceManager.java | 8 +- .../scheduler/AppSchedulingInfo.java | 5 +- .../yarn/server/resourcemanager/MockRM.java | 2 +- .../TestDistributedSchedulingAMService.java | 269 ++++++++++++ .../TestDistributedSchedulingService.java | 271 ------------ 45 files changed, 3029 insertions(+), 2853 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 3bb73f5..6c921cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -301,53 +301,60 @@ public class YarnConfiguration extends Configuration { /** ACL used in case none is found. Allows nothing. */ public static final String DEFAULT_YARN_APP_ACL = " "; - /** Is Distributed Scheduling Enabled. */ + /** Setting that controls whether distributed scheduling is enabled or not. */ public static final String DIST_SCHEDULING_ENABLED = YARN_PREFIX + "distributed-scheduling.enabled"; public static final boolean DIST_SCHEDULING_ENABLED_DEFAULT = false; - /** Mininum allocatable container memory for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_MIN_MEMORY = - YARN_PREFIX + "distributed-scheduling.min-memory"; - public static final int DIST_SCHEDULING_MIN_MEMORY_DEFAULT = 512; - - /** Mininum allocatable container vcores for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_MIN_VCORES = - YARN_PREFIX + "distributed-scheduling.min-vcores"; - public static final int DIST_SCHEDULING_MIN_VCORES_DEFAULT = 1; - - /** Maximum allocatable container memory for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_MAX_MEMORY = - YARN_PREFIX + "distributed-scheduling.max-memory"; - public static final int DIST_SCHEDULING_MAX_MEMORY_DEFAULT = 2048; - - /** Maximum allocatable container vcores for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_MAX_VCORES = - YARN_PREFIX + "distributed-scheduling.max-vcores"; - public static final int DIST_SCHEDULING_MAX_VCORES_DEFAULT = 4; - - /** Incremental allocatable container memory for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_INCR_MEMORY = - YARN_PREFIX + "distributed-scheduling.incr-memory"; - public static final int DIST_SCHEDULING_INCR_MEMORY_DEFAULT = 512; - - /** Incremental allocatable container vcores for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_INCR_VCORES = + /** Minimum memory (in MB) used for allocating a container through distributed + * scheduling. */ + public static final String DIST_SCHEDULING_MIN_CONTAINER_MEMORY_MB = + YARN_PREFIX + "distributed-scheduling.min-container-memory-mb"; + public static final int DIST_SCHEDULING_MIN_CONTAINER_MEMORY_MB_DEFAULT = 512; + + /** Minimum virtual CPU cores used for allocating a container through + * distributed scheduling. */ + public static final String DIST_SCHEDULING_MIN_CONTAINER_VCORES = + YARN_PREFIX + "distributed-scheduling.min-container-vcores"; + public static final int DIST_SCHEDULING_MIN_CONTAINER_VCORES_DEFAULT = 1; + + /** Maximum memory (in MB) used for allocating a container through distributed + * scheduling. */ + public static final String DIST_SCHEDULING_MAX_MEMORY_MB = + YARN_PREFIX + "distributed-scheduling.max-container-memory-mb"; + public static final int DIST_SCHEDULING_MAX_MEMORY_MB_DEFAULT = 2048; + + /** Maximum virtual CPU cores used for allocating a container through + * distributed scheduling. */ + public static final String DIST_SCHEDULING_MAX_CONTAINER_VCORES = + YARN_PREFIX + "distributed-scheduling.max-container-vcores"; + public static final int DIST_SCHEDULING_MAX_CONTAINER_VCORES_DEFAULT = 4; + + /** Incremental memory (in MB) used for allocating a container through + * distributed scheduling. */ + public static final String DIST_SCHEDULING_INCR_CONTAINER_MEMORY_MB = + YARN_PREFIX + "distributed-scheduling.incr-container-memory-mb"; + public static final int DIST_SCHEDULING_INCR_CONTAINER_MEMORY_MB_DEFAULT = + 512; + + /** Incremental virtual CPU cores used for allocating a container through + * distributed scheduling. */ + public static final String DIST_SCHEDULING_INCR_CONTAINER_VCORES = YARN_PREFIX + "distributed-scheduling.incr-vcores"; - public static final int DIST_SCHEDULING_INCR_VCORES_DEFAULT = 1; + public static final int DIST_SCHEDULING_INCR_CONTAINER_VCORES_DEFAULT = 1; - /** Container token expiry for container allocated via Distributed - * Scheduling. */ + /** Container token expiry for container allocated via distributed + * scheduling. */ public static final String DIST_SCHEDULING_CONTAINER_TOKEN_EXPIRY_MS = - YARN_PREFIX + "distributed-scheduling.container-token-expiry"; + YARN_PREFIX + "distributed-scheduling.container-token-expiry-ms"; public static final int DIST_SCHEDULING_CONTAINER_TOKEN_EXPIRY_MS_DEFAULT = 600000; - /** K least loaded nodes to be provided to the LocalScheduler of a - * NodeManager for Distributed Scheduling. */ - public static final String DIST_SCHEDULING_TOP_K = - YARN_PREFIX + "distributed-scheduling.top-k"; - public static final int DIST_SCHEDULING_TOP_K_DEFAULT = 10; + /** Number of nodes to be used by the LocalScheduler of a NodeManager for + * dispatching containers during distributed scheduling. */ + public static final String DIST_SCHEDULING_NODES_NUMBER_USED = + YARN_PREFIX + "distributed-scheduling.nodes-used"; + public static final int DIST_SCHEDULING_NODES_NUMBER_USED_DEFAULT = 10; /** Frequency for computing least loaded NMs. */ public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS = @@ -355,7 +362,7 @@ public class YarnConfiguration extends Configuration { public static final long NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS_DEFAULT = 1000; - /** Comparator for determining Node Load for Distributed Scheduling. */ + /** Comparator for determining node load for Distributed Scheduling. */ public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR = YARN_PREFIX + "nm-container-queuing.load-comparator"; public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR_DEFAULT = @@ -378,13 +385,13 @@ public class YarnConfiguration extends Configuration { YARN_PREFIX + "nm-container-queuing.max-queue-length"; public static final int NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH_DEFAULT = 10; - /** Min wait time of container queue at NodeManager. */ + /** Min queue wait time for a container at a NodeManager. */ public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS = YARN_PREFIX + "nm-container-queuing.min-queue-wait-time-ms"; public static final int NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS_DEFAULT = 1; - /** Max wait time of container queue at NodeManager. */ + /** Max queue wait time for a container queue at a NodeManager. */ public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS = YARN_PREFIX + "nm-container-queuing.max-queue-wait-time-ms"; public static final int NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS_DEFAULT = @@ -1691,17 +1698,21 @@ public class YarnConfiguration extends Configuration { public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX + "application.classpath"; + /** The setting that controls whether AMRMProxy is enabled or not. */ public static final String AMRM_PROXY_ENABLED = NM_PREFIX - + "amrmproxy.enable"; + + "amrmproxy.enabled"; public static final boolean DEFAULT_AMRM_PROXY_ENABLED = false; + public static final String AMRM_PROXY_ADDRESS = NM_PREFIX + "amrmproxy.address"; public static final int DEFAULT_AMRM_PROXY_PORT = 8048; public static final String DEFAULT_AMRM_PROXY_ADDRESS = "0.0.0.0:" + DEFAULT_AMRM_PROXY_PORT; + public static final String AMRM_PROXY_CLIENT_THREAD_COUNT = NM_PREFIX + "amrmproxy.client.thread-count"; public static final int DEFAULT_AMRM_PROXY_CLIENT_THREAD_COUNT = 25; + public static final String AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE = NM_PREFIX + "amrmproxy.interceptor-class.pipeline"; public static final String DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE = http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index a4e5b0a..668821d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -124,41 +124,6 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { configurationPrefixToSkipCompare .add(YarnConfiguration.NM_CPU_RESOURCE_ENABLED); - // Ignore Distributed Scheduling Related Configurations. - // Since it is still a "work in progress" feature - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_ENABLED); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_INCR_MEMORY); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_INCR_VCORES); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_MAX_MEMORY); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_MAX_VCORES); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_CONTAINER_TOKEN_EXPIRY_MS); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_MIN_MEMORY); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_MIN_VCORES); - configurationPrefixToSkipCompare - .add(YarnConfiguration.DIST_SCHEDULING_TOP_K); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_LOAD_COMPARATOR); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS); - configurationPrefixToSkipCompare - .add(YarnConfiguration.NM_CONTAINER_QUEUING_LIMIT_STDEV); - // Set by container-executor.cfg configurationPrefixToSkipCompare.add(YarnConfiguration.NM_USER_HOME_DIR); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java index c649071..71321e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java @@ -82,9 +82,9 @@ import static org.mockito.Mockito.when; /** * Validates End2End Distributed Scheduling flow which includes the AM * specifying OPPORTUNISTIC containers in its resource requests, - * the AMRMProxyService on the NM, the LocalScheduler RequestInterceptor on - * the NM and the DistributedSchedulingProtocol used by the framework to talk - * to the DistributedSchedulingService running on the RM. + * the AMRMProxyService on the NM, the DistributedScheduler RequestInterceptor + * on the NM and the DistributedSchedulingProtocol used by the framework to talk + * to the DistributedSchedulingAMService running on the RM. */ public class TestDistributedScheduling extends BaseAMRMProxyE2ETest { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c8bc741..3ebdc99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2718,10 +2718,10 @@ <property> <description> - Enable/Disable AMRMProxyService in the node manager. This service is used to intercept - calls from the application masters to the resource manager. + Enable/Disable AMRMProxyService in the node manager. This service is used to + intercept calls from the application masters to the resource manager. </description> - <name>yarn.nodemanager.amrmproxy.enable</name> + <name>yarn.nodemanager.amrmproxy.enabled</name> <value>false</value> </property> @@ -2743,8 +2743,9 @@ <property> <description> - The comma separated list of class names that implement the RequestInterceptor interface. This is used by the - AMRMProxyService to create the request processing pipeline for applications. + The comma separated list of class names that implement the + RequestInterceptor interface. This is used by the AMRMProxyService to create + the request processing pipeline for applications. </description> <name>yarn.nodemanager.amrmproxy.interceptor-class.pipeline</name> <value>org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor</value> @@ -2752,6 +2753,141 @@ <property> <description> + Setting that controls whether distributed scheduling is enabled. + </description> + <name>yarn.distributed-scheduling.enabled</name> + <value>false</value> + </property> + + <property> + <description> + Minimum memory (in MB) used for allocating a container through distributed + scheduling. + </description> + <name>yarn.distributed-scheduling.min-container-memory-mb</name> + <value>512</value> + </property> + + <property> + <description> + Minimum virtual CPU cores used for allocating a container through + distributed scheduling. + </description> + <name>yarn.distributed-scheduling.min-container-vcores</name> + <value>1</value> + </property> + + <property> + <description> + Maximum memory (in MB) used for allocating a container through distributed + scheduling. + </description> + <name>yarn.distributed-scheduling.max-container-memory-mb</name> + <value>2048</value> + </property> + + <property> + <description> + Maximum virtual CPU cores used for allocating a container through + distributed scheduling. + </description> + <name>yarn.distributed-scheduling.max-container-vcores</name> + <value>4</value> + </property> + + <property> + <description> + Incremental memory (in MB) used for allocating a container through + distributed scheduling. + </description> + <name>yarn.distributed-scheduling.incr-container-memory-mb</name> + <value>512</value> + </property> + + <property> + <description> + Incremental virtual CPU cores used for allocating a container through + distributed scheduling. + </description> + <name>yarn.distributed-scheduling.incr-vcores</name> + <value>1</value> + </property> + + <property> + <description> + Container token expiry for container allocated via distributed scheduling. + </description> + <name>yarn.distributed-scheduling.container-token-expiry-ms</name> + <value>600000</value> + </property> + + <property> + <description> + Number of nodes to be used by the LocalScheduler of a NodeManager for + dispatching containers during distributed scheduling. + </description> + <name>yarn.distributed-scheduling.nodes-used</name> + <value>10</value> + </property> + + <property> + <description> + Frequency for computing least loaded NMs. + </description> + <name>yarn.nm-container-queuing.sorting-nodes-interval-ms</name> + <value>1000</value> + </property> + + <property> + <description> + Comparator for determining node load for Distributed Scheduling. + </description> + <name>yarn.nm-container-queuing.load-comparator</name> + <value>QUEUE_LENGTH</value> + </property> + + <property> + <description> + Value of standard deviation used for calculation of queue limit thresholds. + </description> + <name>yarn.nm-container-queuing.queue-limit-stdev</name> + <value>1.0f</value> + </property> + + <property> + <description> + Min length of container queue at NodeManager. + </description> + <name>yarn.nm-container-queuing.min-queue-length</name> + <value>1</value> + </property> + + <property> + <description> + Max length of container queue at NodeManager. + </description> + <name>yarn.nm-container-queuing.max-queue-length</name> + <value>10</value> + </property> + + <property> + <description> + Min queue wait time for a container at a NodeManager. + </description> + <name>yarn.nm-container-queuing.min-queue-wait-time-ms</name> + <value>1</value> + </property> + + <property> + <description> + Max queue wait time for a container queue at a NodeManager. + </description> + <name>yarn.nm-container-queuing.max-queue-wait-time-ms</name> + <value>10</value> + </property> + + <property> + <description> Error filename pattern, to identify the file in the container's Log directory which contain the container's error log. As error file redirection is done by client/AM and yarn will not be aware of the error http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index f8330e3..b9e10ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -144,7 +144,7 @@ <source> <directory>${basedir}/src/main/proto</directory> <includes> - <include>distributed_scheduler_protocol.proto</include> + <include>distributed_scheduling_am_protocol.proto</include> <include>yarn_server_common_protos.proto</include> <include>yarn_server_common_service_protos.proto</include> <include>yarn_server_common_service_protos.proto</include> http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java deleted file mode 100644 index 26faa8f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocol.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.io.retry.Idempotent; -import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedAllocateRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedAllocateResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedRegisterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; -import org.apache.hadoop.yarn.exceptions.YarnException; - -import java.io.IOException; - -/** - * <p>This protocol extends the <code>ApplicationMasterProtocol</code>. It is - * used by the <code>LocalScheduler</code> running on the NodeManager to wrap - * the request / response objects of the <code>registerApplicationMaster</code> - * and <code>allocate</code> methods of the protocol with addition information - * required to perform Distributed Scheduling. - * </p> - */ -public interface DistributedSchedulerProtocol - extends ApplicationMasterProtocol { - - /** - * <p> Extends the <code>registerApplicationMaster</code> to wrap the response - * with additional metadata.</p> - * - * @param request ApplicationMaster registration request - * @return A <code>DistSchedRegisterResponse</code> that contains a standard - * AM registration response along with additional information required - * for Distributed Scheduling - * @throws YarnException - * @throws IOException - */ - @Public - @Unstable - @Idempotent - DistSchedRegisterResponse registerApplicationMasterForDistributedScheduling( - RegisterApplicationMasterRequest request) - throws YarnException, IOException; - - /** - * <p> Extends the <code>allocate</code> to wrap the response with additional - * metadata.</p> - * - * @param request ApplicationMaster allocate request - * @return A <code>DistSchedAllocateResponse</code> that contains a standard - * AM allocate response along with additional information required - * for Distributed Scheduling - * @throws YarnException - * @throws IOException - */ - @Public - @Unstable - @Idempotent - DistSchedAllocateResponse allocateForDistributedScheduling( - DistSchedAllocateRequest request) throws YarnException, IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java deleted file mode 100644 index ce7911c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulerProtocolPB.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.ipc.ProtocolInfo; -import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB; -import org.apache.hadoop.yarn.proto.ApplicationMasterProtocol.ApplicationMasterProtocolService; -import org.apache.hadoop.yarn.proto.DistributedSchedulerProtocol.DistributedSchedulerProtocolService; - -@Private -@Unstable -@ProtocolInfo(protocolName = "org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB", - protocolVersion = 1) -public interface DistributedSchedulerProtocolPB extends - DistributedSchedulerProtocolService.BlockingInterface, - ApplicationMasterProtocolService.BlockingInterface, - ApplicationMasterProtocolPB { -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocol.java new file mode 100644 index 0000000..d1ed1fc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocol.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.retry.Idempotent; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterDistributedSchedulingAMResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.io.IOException; + +/** + * <p> + * This protocol extends the <code>ApplicationMasterProtocol</code>. It is used + * by the <code>DistributedScheduler</code> running on the NodeManager to wrap + * the request / response objects of the <code>registerApplicationMaster</code> + * and <code>allocate</code> methods of the protocol with additional information + * required to perform distributed scheduling. + * </p> + */ +public interface DistributedSchedulingAMProtocol + extends ApplicationMasterProtocol { + + /** + * <p> + * Extends the <code>registerApplicationMaster</code> to wrap the response + * with additional metadata. + * </p> + * + * @param request + * ApplicationMaster registration request + * @return A <code>RegisterDistributedSchedulingAMResponse</code> that + * contains a standard AM registration response along with additional + * information required for distributed scheduling + * @throws YarnException YarnException + * @throws IOException IOException + */ + @Public + @Unstable + @Idempotent + RegisterDistributedSchedulingAMResponse + registerApplicationMasterForDistributedScheduling( + RegisterApplicationMasterRequest request) + throws YarnException, IOException; + + /** + * <p> + * Extends the <code>allocate</code> to wrap the response with additional + * metadata. + * </p> + * + * @param request + * ApplicationMaster allocate request + * @return A <code>DistributedSchedulingAllocateResponse</code> that contains + * a standard AM allocate response along with additional information + * required for distributed scheduling + * @throws YarnException YarnException + * @throws IOException IOException + */ + @Public + @Unstable + @Idempotent + DistributedSchedulingAllocateResponse allocateForDistributedScheduling( + DistributedSchedulingAllocateRequest request) + throws YarnException, IOException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocolPB.java new file mode 100644 index 0000000..674d4e4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/DistributedSchedulingAMProtocolPB.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB; +import org.apache.hadoop.yarn.proto.ApplicationMasterProtocol.ApplicationMasterProtocolService; +import org.apache.hadoop.yarn.proto.DistributedSchedulingAMProtocol.DistributedSchedulingAMProtocolService; + +@Private +@Unstable +@ProtocolInfo(protocolName = + "org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB", + protocolVersion = 1) +public interface DistributedSchedulingAMProtocolPB extends + DistributedSchedulingAMProtocolService.BlockingInterface, + ApplicationMasterProtocolService.BlockingInterface, + ApplicationMasterProtocolPB { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java index c23e27c..8555fc3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java @@ -78,7 +78,7 @@ public class ServerRMProxy<T> extends RMProxy<T> { YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); - } else if (protocol == DistributedSchedulerProtocol.class) { + } else if (protocol == DistributedSchedulingAMProtocol.class) { return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulerProtocolPBClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulerProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulerProtocolPBClientImpl.java deleted file mode 100644 index 0ca61df..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulerProtocolPBClientImpl.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.impl.pb.client; - -import com.google.protobuf.ServiceException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos; -import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedAllocateRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedAllocateResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedRegisterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; -import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl; - - -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistSchedAllocateRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistSchedAllocateResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistSchedRegisterResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb - .FinishApplicationMasterRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb - .FinishApplicationMasterResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb - .RegisterApplicationMasterRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb - .RegisterApplicationMasterResponsePBImpl; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServiceProtos; -import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; - -public class DistributedSchedulerProtocolPBClientImpl implements - DistributedSchedulerProtocol, Closeable { - - private DistributedSchedulerProtocolPB proxy; - - public DistributedSchedulerProtocolPBClientImpl(long clientVersion, - InetSocketAddress addr, - Configuration conf) throws IOException { - RPC.setProtocolEngine(conf, DistributedSchedulerProtocolPB.class, - ProtobufRpcEngine.class); - proxy = RPC.getProxy(DistributedSchedulerProtocolPB.class, clientVersion, - addr, conf); - } - - @Override - public void close() { - if (this.proxy != null) { - RPC.stopProxy(this.proxy); - } - } - - @Override - public DistSchedRegisterResponse - registerApplicationMasterForDistributedScheduling( - RegisterApplicationMasterRequest request) - throws YarnException, IOException { - YarnServiceProtos.RegisterApplicationMasterRequestProto requestProto = - ((RegisterApplicationMasterRequestPBImpl) request).getProto(); - try { - return new DistSchedRegisterResponsePBImpl( - proxy.registerApplicationMasterForDistributedScheduling( - null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override - public DistSchedAllocateResponse allocateForDistributedScheduling( - DistSchedAllocateRequest request) throws YarnException, IOException { - YarnServerCommonServiceProtos.DistSchedAllocateRequestProto requestProto = - ((DistSchedAllocateRequestPBImpl) request).getProto(); - try { - return new DistSchedAllocateResponsePBImpl( - proxy.allocateForDistributedScheduling(null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override - public RegisterApplicationMasterResponse registerApplicationMaster - (RegisterApplicationMasterRequest request) throws YarnException, - IOException { - YarnServiceProtos.RegisterApplicationMasterRequestProto requestProto = - ((RegisterApplicationMasterRequestPBImpl) request).getProto(); - try { - return new RegisterApplicationMasterResponsePBImpl( - proxy.registerApplicationMaster(null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override - public FinishApplicationMasterResponse finishApplicationMaster - (FinishApplicationMasterRequest request) throws YarnException, - IOException { - YarnServiceProtos.FinishApplicationMasterRequestProto requestProto = - ((FinishApplicationMasterRequestPBImpl) request).getProto(); - try { - return new FinishApplicationMasterResponsePBImpl( - proxy.finishApplicationMaster(null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } - - @Override - public AllocateResponse allocate(AllocateRequest request) throws - YarnException, IOException { - YarnServiceProtos.AllocateRequestProto requestProto = - ((AllocateRequestPBImpl) request).getProto(); - try { - return new AllocateResponsePBImpl(proxy.allocate(null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulingAMProtocolPBClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulingAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulingAMProtocolPBClientImpl.java new file mode 100644 index 0000000..66893b3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/DistributedSchedulingAMProtocolPBClientImpl.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.impl.pb.client; + +import com.google.protobuf.ServiceException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos; +import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterDistributedSchedulingAMResponse; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistributedSchedulingAMResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB; + +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; + +/** + * Implementation of {@link DistributedSchedulingAMProtocol}, used when + * distributed scheduling is enabled. + */ +public class DistributedSchedulingAMProtocolPBClientImpl implements + DistributedSchedulingAMProtocol, Closeable { + + private DistributedSchedulingAMProtocolPB proxy; + + public DistributedSchedulingAMProtocolPBClientImpl(long clientVersion, + InetSocketAddress addr, Configuration conf) throws IOException { + RPC.setProtocolEngine(conf, DistributedSchedulingAMProtocolPB.class, + ProtobufRpcEngine.class); + proxy = RPC.getProxy(DistributedSchedulingAMProtocolPB.class, clientVersion, + addr, conf); + } + + @Override + public void close() { + if (this.proxy != null) { + RPC.stopProxy(this.proxy); + } + } + + @Override + public RegisterDistributedSchedulingAMResponse + registerApplicationMasterForDistributedScheduling( + RegisterApplicationMasterRequest request) + throws YarnException, IOException { + YarnServiceProtos.RegisterApplicationMasterRequestProto requestProto = + ((RegisterApplicationMasterRequestPBImpl) request).getProto(); + try { + return new RegisterDistributedSchedulingAMResponsePBImpl( + proxy.registerApplicationMasterForDistributedScheduling( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public DistributedSchedulingAllocateResponse allocateForDistributedScheduling( + DistributedSchedulingAllocateRequest request) + throws YarnException, IOException { + YarnServerCommonServiceProtos.DistributedSchedulingAllocateRequestProto + requestProto = + ((DistributedSchedulingAllocateRequestPBImpl) request).getProto(); + try { + return new DistributedSchedulingAllocateResponsePBImpl( + proxy.allocateForDistributedScheduling(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public RegisterApplicationMasterResponse registerApplicationMaster( + RegisterApplicationMasterRequest request) + throws YarnException, IOException { + YarnServiceProtos.RegisterApplicationMasterRequestProto requestProto = + ((RegisterApplicationMasterRequestPBImpl) request).getProto(); + try { + return new RegisterApplicationMasterResponsePBImpl( + proxy.registerApplicationMaster(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public FinishApplicationMasterResponse finishApplicationMaster( + FinishApplicationMasterRequest request) + throws YarnException, IOException { + YarnServiceProtos.FinishApplicationMasterRequestProto requestProto = + ((FinishApplicationMasterRequestPBImpl) request).getProto(); + try { + return new FinishApplicationMasterResponsePBImpl( + proxy.finishApplicationMaster(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public AllocateResponse allocate(AllocateRequest request) + throws YarnException, IOException { + YarnServiceProtos.AllocateRequestProto requestProto = + ((AllocateRequestPBImpl) request).getProto(); + try { + return new AllocateResponsePBImpl(proxy.allocate(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulerProtocolPBServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulerProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulerProtocolPBServiceImpl.java deleted file mode 100644 index 2763259..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulerProtocolPBServiceImpl.java +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.impl.pb.service; - -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocol; -import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos; -import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedAllocateResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.DistSchedRegisterResponse; -import org.apache.hadoop.yarn.api.protocolrecords - .FinishApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistSchedAllocateRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistSchedAllocateResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistSchedRegisterResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb - .FinishApplicationMasterRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb - .FinishApplicationMasterResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto; - -import java.io.IOException; - -public class DistributedSchedulerProtocolPBServiceImpl implements - DistributedSchedulerProtocolPB { - - private DistributedSchedulerProtocol real; - - public DistributedSchedulerProtocolPBServiceImpl( - DistributedSchedulerProtocol impl) { - this.real = impl; - } - - @Override - public YarnServerCommonServiceProtos.DistSchedRegisterResponseProto - registerApplicationMasterForDistributedScheduling(RpcController controller, - RegisterApplicationMasterRequestProto proto) throws - ServiceException { - RegisterApplicationMasterRequestPBImpl request = new - RegisterApplicationMasterRequestPBImpl(proto); - try { - DistSchedRegisterResponse response = - real.registerApplicationMasterForDistributedScheduling(request); - return ((DistSchedRegisterResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } - - @Override - public YarnServerCommonServiceProtos.DistSchedAllocateResponseProto - allocateForDistributedScheduling(RpcController controller, - YarnServerCommonServiceProtos.DistSchedAllocateRequestProto proto) - throws ServiceException { - DistSchedAllocateRequestPBImpl request = - new DistSchedAllocateRequestPBImpl(proto); - try { - DistSchedAllocateResponse response = real - .allocateForDistributedScheduling(request); - return ((DistSchedAllocateResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } - - @Override - public YarnServiceProtos.AllocateResponseProto allocate(RpcController arg0, - AllocateRequestProto proto) throws ServiceException { - AllocateRequestPBImpl request = new AllocateRequestPBImpl(proto); - try { - AllocateResponse response = real.allocate(request); - return ((AllocateResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } - - @Override - public YarnServiceProtos.FinishApplicationMasterResponseProto - finishApplicationMaster( - RpcController arg0, YarnServiceProtos - .FinishApplicationMasterRequestProto proto) - throws ServiceException { - FinishApplicationMasterRequestPBImpl request = new - FinishApplicationMasterRequestPBImpl(proto); - try { - FinishApplicationMasterResponse response = real.finishApplicationMaster - (request); - return ((FinishApplicationMasterResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } - - @Override - public YarnServiceProtos.RegisterApplicationMasterResponseProto - registerApplicationMaster( - RpcController arg0, RegisterApplicationMasterRequestProto proto) - throws ServiceException { - RegisterApplicationMasterRequestPBImpl request = new - RegisterApplicationMasterRequestPBImpl(proto); - try { - RegisterApplicationMasterResponse response = real - .registerApplicationMaster(request); - return ((RegisterApplicationMasterResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulingAMProtocolPBServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulingAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulingAMProtocolPBServiceImpl.java new file mode 100644 index 0000000..68cc077 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/DistributedSchedulingAMProtocolPBServiceImpl.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.impl.pb.service; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocol; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos; +import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterDistributedSchedulingAMResponse; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.DistributedSchedulingAllocateResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RegisterDistributedSchedulingAMResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto; + +import java.io.IOException; + +/** + * Implementation of {@link DistributedSchedulingAMProtocolPB}. + */ +public class DistributedSchedulingAMProtocolPBServiceImpl implements + DistributedSchedulingAMProtocolPB { + + private DistributedSchedulingAMProtocol real; + + public DistributedSchedulingAMProtocolPBServiceImpl( + DistributedSchedulingAMProtocol impl) { + this.real = impl; + } + + @Override + public YarnServerCommonServiceProtos. + RegisterDistributedSchedulingAMResponseProto + registerApplicationMasterForDistributedScheduling( + RpcController controller, RegisterApplicationMasterRequestProto proto) + throws ServiceException { + RegisterApplicationMasterRequestPBImpl request = new + RegisterApplicationMasterRequestPBImpl(proto); + try { + RegisterDistributedSchedulingAMResponse response = + real.registerApplicationMasterForDistributedScheduling(request); + return ((RegisterDistributedSchedulingAMResponsePBImpl) response) + .getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public YarnServerCommonServiceProtos. + DistributedSchedulingAllocateResponseProto + allocateForDistributedScheduling(RpcController controller, + YarnServerCommonServiceProtos. + DistributedSchedulingAllocateRequestProto proto) + throws ServiceException { + DistributedSchedulingAllocateRequestPBImpl request = + new DistributedSchedulingAllocateRequestPBImpl(proto); + try { + DistributedSchedulingAllocateResponse response = real + .allocateForDistributedScheduling(request); + return ((DistributedSchedulingAllocateResponsePBImpl) response) + .getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public YarnServiceProtos.AllocateResponseProto allocate(RpcController arg0, + AllocateRequestProto proto) throws ServiceException { + AllocateRequestPBImpl request = new AllocateRequestPBImpl(proto); + try { + AllocateResponse response = real.allocate(request); + return ((AllocateResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public YarnServiceProtos.FinishApplicationMasterResponseProto + finishApplicationMaster( + RpcController arg0, YarnServiceProtos + .FinishApplicationMasterRequestProto proto) + throws ServiceException { + FinishApplicationMasterRequestPBImpl request = new + FinishApplicationMasterRequestPBImpl(proto); + try { + FinishApplicationMasterResponse response = real.finishApplicationMaster + (request); + return ((FinishApplicationMasterResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public YarnServiceProtos.RegisterApplicationMasterResponseProto + registerApplicationMaster( + RpcController arg0, RegisterApplicationMasterRequestProto proto) + throws ServiceException { + RegisterApplicationMasterRequestPBImpl request = new + RegisterApplicationMasterRequestPBImpl(proto); + try { + RegisterApplicationMasterResponse response = real + .registerApplicationMaster(request); + return ((RegisterApplicationMasterResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateRequest.java deleted file mode 100644 index 10ff95b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateRequest.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; -import org.apache.hadoop.yarn.api.records.Container; - -import java.util.List; - -/** - * Request for a distributed scheduler to notify allocation of containers to - * the Resource Manager. - */ -@Public -@Evolving -public abstract class DistSchedAllocateRequest { - - /** - * Get the underlying <code>AllocateRequest</code> object. - * @return Allocate request - */ - @Public - @Evolving - public abstract AllocateRequest getAllocateRequest(); - - /** - * Set the underlying <code>AllocateRequest</code> object. - * @param allocateRequest Allocate request - */ - @Public - @Evolving - public abstract void setAllocateRequest(AllocateRequest allocateRequest); - - /** - * Get the list of <em>newly allocated</em> <code>Container</code> by the - * Distributed Scheduling component on the NodeManager. - * @return list of <em>newly allocated</em> <code>Container</code> - */ - @Public - @Evolving - public abstract List<Container> getAllocatedContainers(); - - /** - * Set the list of <em>newly allocated</em> <code>Container</code> by the - * Distributed Scheduling component on the NodeManager. - * @param containers list of <em>newly allocated</em> <code>Container</code> - */ - @Public - @Evolving - public abstract void setAllocatedContainers(List<Container> containers); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateResponse.java deleted file mode 100644 index 5f6e069..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedAllocateResponse.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.util.Records; - -import java.util.List; - -@Public -@Unstable -public abstract class DistSchedAllocateResponse { - - @Public - @Unstable - public static DistSchedAllocateResponse newInstance(AllocateResponse - allResp) { - DistSchedAllocateResponse response = - Records.newRecord(DistSchedAllocateResponse.class); - response.setAllocateResponse(allResp); - return response; - } - - @Public - @Unstable - public abstract void setAllocateResponse(AllocateResponse response); - - @Public - @Unstable - public abstract AllocateResponse getAllocateResponse(); - - @Public - @Unstable - public abstract void setNodesForScheduling(List<NodeId> nodesForScheduling); - - @Public - @Unstable - public abstract List<NodeId> getNodesForScheduling(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedRegisterResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedRegisterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedRegisterResponse.java deleted file mode 100644 index e4e5138..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistSchedRegisterResponse.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords - .RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.Records; - -import java.util.List; - -@Public -@Unstable -public abstract class DistSchedRegisterResponse { - - @Public - @Unstable - public static DistSchedRegisterResponse newInstance - (RegisterApplicationMasterResponse regAMResp) { - DistSchedRegisterResponse response = - Records.newRecord(DistSchedRegisterResponse.class); - response.setRegisterResponse(regAMResp); - return response; - } - - @Public - @Unstable - public abstract void setRegisterResponse( - RegisterApplicationMasterResponse resp); - - @Public - @Unstable - public abstract RegisterApplicationMasterResponse getRegisterResponse(); - - @Public - @Unstable - public abstract void setMinAllocatableCapabilty(Resource minResource); - - @Public - @Unstable - public abstract Resource getMinAllocatableCapabilty(); - - @Public - @Unstable - public abstract void setMaxAllocatableCapabilty(Resource maxResource); - - @Public - @Unstable - public abstract Resource getMaxAllocatableCapabilty(); - - @Public - @Unstable - public abstract void setIncrAllocatableCapabilty(Resource maxResource); - - @Public - @Unstable - public abstract Resource getIncrAllocatableCapabilty(); - - @Public - @Unstable - public abstract void setContainerTokenExpiryInterval(int interval); - - @Public - @Unstable - public abstract int getContainerTokenExpiryInterval(); - - @Public - @Unstable - public abstract void setContainerIdStart(long containerIdStart); - - @Public - @Unstable - public abstract long getContainerIdStart(); - - @Public - @Unstable - public abstract void setNodesForScheduling(List<NodeId> nodesForScheduling); - - @Public - @Unstable - public abstract List<NodeId> getNodesForScheduling(); - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateRequest.java new file mode 100644 index 0000000..ac40592 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateRequest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.records.Container; + +import java.util.List; + +/** + * Object used by the Application Master when distributed scheduling is enabled, + * in order to forward the {@link AllocateRequest} for GUARANTEED containers to + * the Resource Manager, and to notify the Resource Manager about the allocation + * of OPPORTUNISTIC containers through the Distributed Scheduler. + */ +@Public +@Evolving +public abstract class DistributedSchedulingAllocateRequest { + + /** + * Get the underlying <code>AllocateRequest</code> object. + * @return Allocate request + */ + @Public + @Evolving + public abstract AllocateRequest getAllocateRequest(); + + /** + * Set the underlying <code>AllocateRequest</code> object. + * @param allocateRequest Allocate request + */ + @Public + @Evolving + public abstract void setAllocateRequest(AllocateRequest allocateRequest); + + /** + * Get the list of <em>newly allocated</em> <code>Container</code> by the + * Distributed Scheduling component on the NodeManager. + * @return list of <em>newly allocated</em> <code>Container</code> + */ + @Public + @Evolving + public abstract List<Container> getAllocatedContainers(); + + /** + * Set the list of <em>newly allocated</em> <code>Container</code> by the + * Distributed Scheduling component on the NodeManager. + * @param containers list of <em>newly allocated</em> <code>Container</code> + */ + @Public + @Evolving + public abstract void setAllocatedContainers(List<Container> containers); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5766b1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java new file mode 100644 index 0000000..7a40449 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/DistributedSchedulingAllocateResponse.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * This is the response of the Resource Manager to the + * {@link DistributedSchedulingAllocateRequest}, when distributed scheduling is + * enabled. It includes the {@link AllocateResponse} for the GUARANTEED + * containers allocated by the Resource Manager. Moreover, it includes a list + * with the nodes that can be used by the Distributed Scheduler when allocating + * containers. + */ +@Public +@Unstable +public abstract class DistributedSchedulingAllocateResponse { + + @Public + @Unstable + public static DistributedSchedulingAllocateResponse newInstance( + AllocateResponse allResp) { + DistributedSchedulingAllocateResponse response = + Records.newRecord(DistributedSchedulingAllocateResponse.class); + response.setAllocateResponse(allResp); + return response; + } + + @Public + @Unstable + public abstract void setAllocateResponse(AllocateResponse response); + + @Public + @Unstable + public abstract AllocateResponse getAllocateResponse(); + + @Public + @Unstable + public abstract void setNodesForScheduling(List<NodeId> nodesForScheduling); + + @Public + @Unstable + public abstract List<NodeId> getNodesForScheduling(); +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
