MAPREDUCE-6353. Divide by zero error in MR AM when calculating available containers. (Anubhav Dhoot via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9a43fa6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9a43fa6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9a43fa6 Branch: refs/heads/YARN-2928 Commit: a9a43fa69c61719c1541688211bd5148121788a7 Parents: ca59e77 Author: Karthik Kambatla <ka...@apache.org> Authored: Sat May 9 14:43:18 2015 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Tue May 12 13:24:11 2015 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/rm/ResourceCalculatorUtils.java | 16 ++++- .../v2/app/rm/TestResourceCalculatorUtils.java | 75 ++++++++++++++++++++ 3 files changed, 91 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a43fa6/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ea1a148..2152be0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -423,6 +423,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6359. In RM HA setup, "Cluster" tab links populated with AM hostname instead of RM. (zhaoyunjiong via junping_du) + MAPREDUCE-6353. Divide by zero error in MR AM when calculating available + containers. (Anubhav Dhoot via kasha) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a43fa6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java index b9bc8b5..39cb22e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ResourceCalculatorUtils.java @@ -35,10 +35,13 @@ public class ResourceCalculatorUtils { public static int computeAvailableContainers(Resource available, Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) { if (resourceTypes.contains(SchedulerResourceTypes.CPU)) { - return Math.min(available.getMemory() / required.getMemory(), - available.getVirtualCores() / required.getVirtualCores()); + return Math.min( + calculateRatioOrMaxValue(available.getMemory(), required.getMemory()), + calculateRatioOrMaxValue(available.getVirtualCores(), required + .getVirtualCores())); } - return available.getMemory() / required.getMemory(); + return calculateRatioOrMaxValue( + available.getMemory(), required.getMemory()); } public static int divideAndCeilContainers(Resource required, Resource factor, @@ -49,4 +52,11 @@ public class ResourceCalculatorUtils { } return divideAndCeil(required.getMemory(), factor.getMemory()); } + + private static int calculateRatioOrMaxValue(int numerator, int denominator) { + if (denominator == 0) { + return Integer.MAX_VALUE; + } + return numerator / denominator; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a43fa6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java new file mode 100644 index 0000000..d87f6db --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestResourceCalculatorUtils.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.app.rm; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.junit.Assert; +import org.junit.Test; + +import java.util.EnumSet; + +import static org.apache.hadoop.yarn.proto.YarnServiceProtos.*; + +public class TestResourceCalculatorUtils { + @Test + public void testComputeAvailableContainers() throws Exception { + Resource clusterAvailableResources = Resource.newInstance(81920, 40); + + Resource nonZeroResource = Resource.newInstance(1024, 2); + + int expectedNumberOfContainersForMemory = 80; + int expectedNumberOfContainersForCPU = 20; + + verifyDifferentResourceTypes(clusterAvailableResources, nonZeroResource, + expectedNumberOfContainersForMemory, + expectedNumberOfContainersForCPU); + + Resource zeroMemoryResource = Resource.newInstance(0, + nonZeroResource.getVirtualCores()); + + verifyDifferentResourceTypes(clusterAvailableResources, zeroMemoryResource, + Integer.MAX_VALUE, + expectedNumberOfContainersForCPU); + + Resource zeroCpuResource = Resource.newInstance(nonZeroResource.getMemory(), + 0); + + verifyDifferentResourceTypes(clusterAvailableResources, zeroCpuResource, + expectedNumberOfContainersForMemory, + expectedNumberOfContainersForMemory); + } + + private void verifyDifferentResourceTypes(Resource clusterAvailableResources, + Resource nonZeroResource, int expectedNumberOfContainersForMemoryOnly, + int expectedNumberOfContainersOverall) { + + Assert.assertEquals("Incorrect number of available containers for Memory", + expectedNumberOfContainersForMemoryOnly, + ResourceCalculatorUtils.computeAvailableContainers( + clusterAvailableResources, nonZeroResource, + EnumSet.of(SchedulerResourceTypes.MEMORY))); + + Assert.assertEquals("Incorrect number of available containers overall", + expectedNumberOfContainersOverall, + ResourceCalculatorUtils.computeAvailableContainers( + clusterAvailableResources, nonZeroResource, + EnumSet.of(SchedulerResourceTypes.CPU, + SchedulerResourceTypes.MEMORY))); + } +}