YARN-6066. Opportunistic containers Minor fixes : API annotations, parameter name changes, checkstyles. (asuresh)
(cherry picked from commit 4985217de453a04ddffd7b52644bdc8d153f753c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85826f6c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85826f6c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85826f6c Branch: refs/heads/HADOOP-13345 Commit: 85826f6ca5a6d06b711a6805f7a1a6788852db05 Parents: 679478d Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 30 08:46:10 2016 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Sun Jan 8 09:04:20 2017 -0800 ---------------------------------------------------------------------- .../v2/app/rm/RMContainerAllocator.java | 20 ++++++++++---------- .../apache/hadoop/mapreduce/MRJobConfig.java | 6 +++--- .../hadoop/mapred/TestMROpportunisticMaps.java | 2 +- .../hadoop/yarn/api/records/ContainerState.java | 2 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++++ .../site/markdown/OpportunisticContainers.md | 4 ++-- 6 files changed, 27 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 2b63b9e..31bc380 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -237,10 +237,10 @@ public class RMContainerAllocator extends RMContainerRequestor // Init startTime to current time. If all goes well, it will be reset after // first attempt to contact RM. retrystartTime = System.currentTimeMillis(); - this.scheduledRequests.setNumOpportunisticMapsPer100( - conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, - MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE)); - LOG.info(this.scheduledRequests.getNumOpportunisticMapsPer100() + + this.scheduledRequests.setNumOpportunisticMapsPercent( + conf.getInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, + MRJobConfig.DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT)); + LOG.info(this.scheduledRequests.getNumOpportunisticMapsPercent() + "% of the mappers will be scheduled using OPPORTUNISTIC containers"); } @@ -1056,14 +1056,14 @@ public class RMContainerAllocator extends RMContainerRequestor final Map<TaskAttemptId, ContainerRequest> maps = new LinkedHashMap<TaskAttemptId, ContainerRequest>(); int mapsMod100 = 0; - int numOpportunisticMapsPer100 = 0; + int numOpportunisticMapsPercent = 0; - void setNumOpportunisticMapsPer100(int numMaps) { - this.numOpportunisticMapsPer100 = numMaps; + void setNumOpportunisticMapsPercent(int numMaps) { + this.numOpportunisticMapsPercent = numMaps; } - int getNumOpportunisticMapsPer100() { - return this.numOpportunisticMapsPer100; + int getNumOpportunisticMapsPercent() { + return this.numOpportunisticMapsPercent; } @VisibleForTesting @@ -1110,7 +1110,7 @@ public class RMContainerAllocator extends RMContainerRequestor maps.put(event.getAttemptID(), request); addContainerReq(request); } else { - if (mapsMod100 < numOpportunisticMapsPer100) { + if (mapsMod100 < numOpportunisticMapsPercent) { request = new ContainerRequest(event, PRIORITY_OPPORTUNISTIC_MAP, mapNodeLabelExpression); http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 32bbe96..7add7df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -1005,9 +1005,9 @@ public interface MRJobConfig { * requested by the AM will be opportunistic. If the total number of maps * for the job is less than 'x', then ALL maps will be OPPORTUNISTIC */ - public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = - "mapreduce.job.num-opportunistic-maps-percentage"; - public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE = 0; + public static final String MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = + "mapreduce.job.num-opportunistic-maps-percent"; + public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0; /** * A comma-separated list of properties whose value will be redacted. http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java index 462ff04..eed731f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMROpportunisticMaps.java @@ -145,7 +145,7 @@ public class TestMROpportunisticMaps { job.setNumReduceTasks(numReducers); // All OPPORTUNISTIC - job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENTAGE, percent); + job.setInt(MRJobConfig.MR_NUM_OPPORTUNISTIC_MAPS_PERCENT, percent); job.setInt("mapreduce.map.maxattempts", 1); job.setInt("mapreduce.reduce.maxattempts", 1); job.setInt("mapred.test.num_lines", numLines); http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java index 4efd8c1..696fe06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records; import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability.Stable; /** @@ -37,5 +38,6 @@ public enum ContainerState { COMPLETE, /** Scheduled (awaiting resources) at the NM. */ + @InterfaceStability.Unstable SCHEDULED } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d0ade22..7dd5ce3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -314,6 +314,7 @@ public class YarnConfiguration extends Configuration { /** Setting that controls whether opportunistic container allocation * is enabled or not. */ + @Unstable public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED = RM_PREFIX + "opportunistic-container-allocation.enabled"; public static final boolean @@ -321,12 +322,14 @@ public class YarnConfiguration extends Configuration { /** Number of nodes to be used by the Opportunistic Container allocator for * dispatching containers during container allocation. */ + @Unstable public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED = RM_PREFIX + "opportunistic-container-allocation.nodes-used"; public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED = 10; /** Frequency for computing least loaded NMs. */ + @Unstable public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS = RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms"; public static final long @@ -334,6 +337,7 @@ public class YarnConfiguration extends Configuration { /** Comparator for determining node load for scheduling of opportunistic * containers. */ + @Unstable public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR = RM_PREFIX + "nm-container-queuing.load-comparator"; public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR = @@ -341,6 +345,7 @@ public class YarnConfiguration extends Configuration { /** Value of standard deviation used for calculation of queue limit * thresholds. */ + @Unstable public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV = RM_PREFIX + "nm-container-queuing.queue-limit-stdev"; public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV = @@ -349,6 +354,7 @@ public class YarnConfiguration extends Configuration { /** Min length of container queue at NodeManager. This is a cluster-wide * configuration that acts as the lower-bound of optimal queue length * calculated by the NodeQueueLoadMonitor */ + @Unstable public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = RM_PREFIX + "nm-container-queuing.min-queue-length"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5; @@ -356,17 +362,20 @@ public class YarnConfiguration extends Configuration { /** Max length of container queue at NodeManager. This is a cluster-wide * configuration that acts as the upper-bound of optimal queue length * calculated by the NodeQueueLoadMonitor */ + @Unstable public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = RM_PREFIX + "nm-container-queuing.max-queue-length"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15; /** Min queue wait time for a container at a NodeManager. */ + @Unstable public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS = RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS = 10; /** Max queue wait time for a container queue at a NodeManager. */ + @Unstable public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS = RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms"; public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS = http://git-wip-us.apache.org/repos/asf/hadoop/blob/85826f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md index ac26d88..223930e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/OpportunisticContainers.md @@ -72,10 +72,10 @@ By default, allocation of opportunistic containers is performed centrally throug The following command can be used to run a sample pi map-reduce job, executing 40% of mappers using opportunistic containers (substitute `3.0.0-alpha2-SNAPSHOT` below with the version of Hadoop you are using): ``` -$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percentage="40" 50 100 +$ hadoop jar hadoop-3.0.0-alpha2-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0-alpha2-SNAPSHOT.jar pi -Dmapreduce.job.num-opportunistic-maps-percent="40" 50 100 ``` -By changing the value of `mapreduce.job.num-opportunistic-maps-percentage` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers. +By changing the value of `mapreduce.job.num-opportunistic-maps-percent` in the above command, we can specify the percentage of mappers that can be executed through opportunistic containers. ###<a name="Opportunistic_Containers_in_Web_UI"></a>Opportunistic Containers in Web UI --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
