Author: tucu Date: Fri Jun 14 13:43:25 2013 New Revision: 1493065 URL: http://svn.apache.org/r1493065 Log: YARN-782. vcores-pcores ratio functions differently from vmem-pmem ratio in misleading way. (sandyr via tucu)
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1493065&r1=1493064&r2=1493065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Jun 14 13:43:25 2013 @@ -317,6 +317,9 @@ Release 2.1.0-beta - UNRELEASED YARN-692. Creating NMToken master key on RM and sharing it with NM as a part of RM-NM heartbeat. (Omkar Vinit Joshi via vinodkv) + YARN-782. vcores-pcores ratio functions differently from vmem-pmem ratio in + misleading way. (sandyr via tucu) + OPTIMIZATIONS YARN-512. Log aggregation root directory check is more expensive than it Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1493065&r1=1493064&r2=1493065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Jun 14 13:43:25 2013 @@ -470,14 +470,9 @@ public class YarnConfiguration extends C NM_PREFIX + "vmem-pmem-ratio"; public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; - /** Number of Physical CPU Cores which can be allocated for containers.*/ - public static final String NM_VCORES = NM_PREFIX + "resource.cpu-cores"; + /** Number of Virtual CPU Cores which can be allocated for containers.*/ + public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores"; public static final int DEFAULT_NM_VCORES = 8; - - /** Conversion ratio for physical cores to virtual cores. */ - public static final String NM_VCORES_PCORES_RATIO = - NM_PREFIX + "vcores-pcores-ratio"; - public static final float DEFAULT_NM_VCORES_PCORES_RATIO = 2.0f; /** NM Webapp address.**/ public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1493065&r1=1493064&r2=1493065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Jun 14 13:43:25 2013 @@ -510,19 +510,11 @@ <property> <description>Number of CPU cores that can be allocated for containers.</description> - <name>yarn.nodemanager.resource.cpu-cores</name> + <name>yarn.nodemanager.resource.cpu-vcores</name> <value>8</value> </property> <property> - <description>Ratio between virtual cores to physical cores when - allocating CPU resources to containers. - </description> - <name>yarn.nodemanager.vcores-pcores-ratio</name> - <value>2</value> - </property> - - <property> <description>NM Webapp address.</description> <name>yarn.nodemanager.webapp.address</name> <value>${yarn.nodemanager.hostname}:8042</value> Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1493065&r1=1493064&r2=1493065&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Jun 14 13:43:25 2013 @@ -124,14 +124,9 @@ public class NodeStatusUpdaterImpl exten YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); int virtualMemoryMb = (int)Math.ceil(memoryMb * vMemToPMem); - int cpuCores = + int virtualCores = conf.getInt( YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES); - float vCoresToPCores = - conf.getFloat( - YarnConfiguration.NM_VCORES_PCORES_RATIO, - YarnConfiguration.DEFAULT_NM_VCORES_PCORES_RATIO); - int virtualCores = (int)Math.ceil(cpuCores * vCoresToPCores); this.totalResource = recordFactory.newRecordInstance(Resource.class); this.totalResource.setMemory(memoryMb); @@ -144,7 +139,7 @@ public class NodeStatusUpdaterImpl exten LOG.info("Initialized nodemanager for " + nodeId + ":" + " physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb + - " physical-cores=" + cpuCores + " virtual-cores=" + virtualCores); + " virtual-cores=" + virtualCores); super.serviceInit(conf); }