http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt ---------------------------------------------------------------------- diff --git a/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt b/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000..63fbc9d --- /dev/null +++ b/hadoop-build-tools/src/main/resources/META-INF/NOTICE.txt @@ -0,0 +1,283 @@ +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +The binary distribution of this product bundles binaries of +org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the +following notices: +* Copyright 2011 Dain Sundstrom <d...@iq80.com> +* Copyright 2011 FuseSource Corp. http://fusesource.com + +The binary distribution of this product bundles binaries of +org.fusesource.hawtjni:hawtjni-runtime (https://github.com/fusesource/hawtjni), +which has the following notices: +* This product includes software developed by FuseSource Corp. + http://fusesource.com +* This product includes software developed at + Progress Software Corporation and/or its subsidiaries or affiliates. +* This product includes software developed by IBM Corporation and others. + +The binary distribution of this product bundles binaries of +AWS Java SDK 1.10.6, +which has the following notices: + * This software includes third party software subject to the following + copyrights: - XML parsing and utility functions from JetS3t - Copyright + 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - + Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility + functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. + +The binary distribution of this product bundles binaries of +Gson 2.2.4, +which has the following notices: + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2014 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +Also, please refer to each LICENSE.<component>.txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified portion of 'Webbit', an event based +WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + +This product contains a modified portion of 'SLF4J', a simple logging +facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product contains a modified portion of 'ArrayDeque', written by Josh +Bloch of Google, Inc: + + * LICENSE: + * license/LICENSE.deque.txt (Public Domain) + +This product contains a modified portion of 'Apache Harmony', an open source +Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + +This product contains a modified version of Roland Kuhn's ASL2 +AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. +It can be obtained at: + + * LICENSE: + * license/LICENSE.abstractnodequeue.txt (Public Domain) + * HOMEPAGE: + * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java + +This product contains a modified portion of 'jbzip2', a Java bzip2 compression +and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + +This product contains a modified portion of 'libdivsufsort', a C API library to construct +the suffix array and the Burrows-Wheeler transformed string for any input string of +a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/libdivsufsort/ + +This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + +This product optionally depends on 'JZlib', a re-implementation of zlib in +pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Compress-LZF', a Java library for encoding and +decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + +This product optionally depends on 'lz4', a LZ4 Java compression +and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + +This product optionally depends on 'lzma-java', a LZMA Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + +This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression +and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + +This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'Bouncy Castle Crypto APIs' to generate +a temporary self-signed X.509 certificate when the JVM does not provide the +equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + +This product optionally depends on 'Snappy', a compression library produced +by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/snappy/ + +This product optionally depends on 'JBoss Marshalling', an alternative Java +serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + +This product optionally depends on 'Caliper', Google's micro- +benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * http://code.google.com/p/caliper/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, which +can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'Aalto XML', an ultra-high performance +non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + +This product contains a modified portion of 'Apache Commons Lang', a Java library +provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + +The binary distribution of this product bundles binaries of +Commons Codec 1.4, +which has the following notices: + * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kev...@gnu.org) + =============================================================================== + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +The binary distribution of this product bundles binaries of +Commons Lang 2.6, +which has the following notices: + * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + +The binary distribution of this product bundles binaries of +Apache Log4j 1.2.17, +which has the following notices: + * ResolverUtil.java + Copyright 2005-2006 Tim Fennell + Dumbster SMTP test server + Copyright 2004 Jason Paul Kitchen + TypeUtil.java + Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java new file mode 100644 index 0000000..dbd9c37 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceTypes.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +/** + * Enum which represents the resource type. Currently, the only type allowed is + * COUNTABLE. + */ +public enum ResourceTypes { + COUNTABLE +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 0d31b6f..bbf61aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -23,8 +23,11 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; +import java.util.Map; /** * <p><code>Resource</code> models a set of computer resources in the @@ -39,10 +42,10 @@ import org.apache.hadoop.yarn.util.Records; * the average number of threads it expects to have runnable at a time.</p> * * <p>Virtual cores take integer values and thus currently CPU-scheduling is - * very coarse. A complementary axis for CPU requests that represents processing - * power will likely be added in the future to enable finer-grained resource - * configuration.</p> - * + * very coarse. A complementary axis for CPU requests that represents + * processing power will likely be added in the future to enable finer-grained + * resource configuration.</p> + * * <p>Typically, applications request <code>Resource</code> of suitable * capability to run their component tasks.</p> * @@ -71,19 +74,40 @@ public abstract class Resource implements Comparable<Resource> { return resource; } + @Public + @Stable + public static Resource newInstance( + Map<String, ResourceInformation> resources) { + Resource resource = Records.newRecord(Resource.class); + resource.setResources(resources); + return resource; + } + /** * This method is DEPRECATED: * Use {@link Resource#getMemorySize()} instead * - * Get <em>memory</em> of the resource. - * @return <em>memory</em> of the resource + * Get <em>memory</em> of the resource. Note - while memory has + * never had a unit specified, all YARN configurations have specified memory + * in MB. The assumption has been that the daemons and applications are always + * using the same units. With the introduction of the ResourceInformation + * class we have support for units - so this function will continue to return + * memory but in the units of MB + * + * @return <em>memory</em>(in MB) of the resource */ @Public @Deprecated public abstract int getMemory(); /** - * Get <em>memory</em> of the resource. + * Get <em>memory</em> of the resource. Note - while memory has + * never had a unit specified, all YARN configurations have specified memory + * in MB. The assumption has been that the daemons and applications are always + * using the same units. With the introduction of the ResourceInformation + * class we have support for units - so this function will continue to return + * memory but in the units of MB + * * @return <em>memory</em> of the resource */ @Public @@ -94,8 +118,14 @@ public abstract class Resource implements Comparable<Resource> { } /** - * Set <em>memory</em> of the resource. - * @param memory <em>memory</em> of the resource + * Set <em>memory</em> of the resource. Note - while memory has + * never had a unit specified, all YARN configurations have specified memory + * in MB. The assumption has been that the daemons and applications are always + * using the same units. With the introduction of the ResourceInformation + * class we have support for units - so this function will continue to set + * memory but the assumption is that the value passed is in units of MB. + * + * @param memory <em>memory</em>(in MB) of the resource */ @Public @Deprecated @@ -117,10 +147,11 @@ public abstract class Resource implements Comparable<Resource> { * Get <em>number of virtual cpu cores</em> of the resource. * * Virtual cores are a unit for expressing CPU parallelism. A node's capacity - * should be configured with virtual cores equal to its number of physical cores. - * A container should be requested with the number of cores it can saturate, i.e. - * the average number of threads it expects to have runnable at a time. - * + * should be configured with virtual cores equal to its number of physical + * cores. A container should be requested with the number of cores it can + * saturate, i.e. the average number of threads it expects to have runnable + * at a time. + * * @return <em>num of virtual cpu cores</em> of the resource */ @Public @@ -131,16 +162,84 @@ public abstract class Resource implements Comparable<Resource> { * Set <em>number of virtual cpu cores</em> of the resource. * * Virtual cores are a unit for expressing CPU parallelism. A node's capacity - * should be configured with virtual cores equal to its number of physical cores. - * A container should be requested with the number of cores it can saturate, i.e. - * the average number of threads it expects to have runnable at a time. - * + * should be configured with virtual cores equal to its number of physical + * cores. A container should be requested with the number of cores it can + * saturate, i.e. the average number of threads it expects to have runnable + * at a time. + * * @param vCores <em>number of virtual cpu cores</em> of the resource */ @Public @Evolving public abstract void setVirtualCores(int vCores); + /** + * Get ResourceInformation for all resources. + * + * @return Map of resource name to ResourceInformation + */ + @Public + @Evolving + public abstract Map<String, ResourceInformation> getResources(); + + /** + * Get ResourceInformation for a specified resource. + * + * @param resource name of the resource + * @return the ResourceInformation object for the resource + * @throws YarnException if the resource can't be found + */ + @Public + @Evolving + public abstract ResourceInformation getResourceInformation(String resource) + throws YarnException; + + /** + * Get the value for a specified resource. No information about the units is + * returned. + * + * @param resource name of the resource + * @return the value for the resource + * @throws YarnException if the resource can't be found + */ + @Public + @Evolving + public abstract Long getResourceValue(String resource) throws YarnException; + + /** + * Set the resources to the map specified. + * + * @param resources Desired resources + */ + @Public + @Evolving + public abstract void setResources(Map<String, ResourceInformation> resources); + + /** + * Set the ResourceInformation object for a particular resource. + * + * @param resource the resource for which the ResourceInformation is provided + * @param resourceInformation ResourceInformation object + * @throws ResourceNotFoundException if the resource is not found + */ + @Public + @Evolving + public abstract void setResourceInformation(String resource, + ResourceInformation resourceInformation) throws ResourceNotFoundException; + + /** + * Set the value of a resource in the ResourceInformation object. The unit of + * the value is assumed to be the one in the ResourceInformation object. + * + * @param resource the resource for which the value is provided. + * @param value the value to set + * @throws ResourceNotFoundException if the resource is not found + */ + @Public + @Evolving + public abstract void setResourceValue(String resource, Long value) + throws ResourceNotFoundException; + @Override public int hashCode() { final int prime = 263167; @@ -148,27 +247,84 @@ public abstract class Resource implements Comparable<Resource> { int result = (int) (939769357 + getMemorySize()); // prime * result = 939769357 initially result = prime * result + getVirtualCores(); + for (Map.Entry<String, ResourceInformation> entry : getResources() + .entrySet()) { + if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) || entry + .getKey().equals(ResourceInformation.VCORES.getName())) { + continue; + } + result = prime * result + entry.getValue().hashCode(); + } return result; } @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (!(obj instanceof Resource)) + } + if (!(obj instanceof Resource)) { return false; + } Resource other = (Resource) obj; - if (getMemorySize() != other.getMemorySize() || - getVirtualCores() != other.getVirtualCores()) { + if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other + .getVirtualCores()) { return false; } - return true; + return this.getResources().equals(other.getResources()); } @Override public String toString() { - return "<memory:" + getMemorySize() + ", vCores:" + getVirtualCores() + ">"; + StringBuilder sb = new StringBuilder(); + sb.append("<memory:").append(getMemorySize()).append(", vCores:") + .append(getVirtualCores()); + for (Map.Entry<String, ResourceInformation> entry : getResources() + .entrySet()) { + if (entry.getKey().equals(ResourceInformation.MEMORY.getName()) + && entry.getValue().getUnits() + .equals(ResourceInformation.MEMORY_MB.getUnits())) { + continue; + } + if (entry.getKey().equals(ResourceInformation.VCORES.getName()) + && entry.getValue().getUnits().equals("")) { + continue; + } + sb.append(", ").append(entry.getKey()).append(": ") + .append(entry.getValue().getValue()) + .append(entry.getValue().getUnits()); + } + sb.append(">"); + return sb.toString(); + } + + @Override + public int compareTo(Resource other) { + Map<String, ResourceInformation> thisResources, otherResources; + thisResources = this.getResources(); + otherResources = other.getResources(); + long diff = thisResources.size() - otherResources.size(); + if (diff == 0) { + if (thisResources.keySet().equals(otherResources.keySet())) { + diff = this.getMemorySize() - other.getMemorySize(); + if (diff == 0) { + diff = this.getVirtualCores() - other.getVirtualCores(); + } + if (diff == 0) { + for (Map.Entry<String, ResourceInformation> entry : thisResources + .entrySet()) { + diff = + entry.getValue().compareTo(otherResources.get(entry.getKey())); + if (diff != 0) { + break; + } + } + } + } + } + return Long.compare(diff, 0); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java new file mode 100644 index 0000000..4e780c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +/** + * Class to encapsulate information about a Resource - the name of the resource, + * the units(milli, micro, etc), the type(countable), and the value. + */ +public class ResourceInformation implements Comparable<ResourceInformation> { + + private String name; + private String units; + private ResourceTypes resourceType; + private Long value; + + private static final String MEMORY_URI = "yarn.io/memory"; + private static final String VCORES_URI = "yarn.io/vcores"; + + public static final ResourceInformation MEMORY = + ResourceInformation.newInstance(MEMORY_URI); + public static final ResourceInformation MEMORY_MB = + ResourceInformation.newInstance(MEMORY_URI, "M"); + public static final ResourceInformation VCORES = + ResourceInformation.newInstance(VCORES_URI); + + /** + * Get the name for the resource. + * + * @return resource name + */ + public String getName() { + return name; + } + + /** + * Set the name for the resource. + * + * @param rName name for the resource + */ + public void setName(String rName) { + this.name = rName; + } + + /** + * Get units for the resource. + * + * @return units for the resource + */ + public String getUnits() { + return units; + } + + /** + * Set the units for the resource. + * + * @param rUnits units for the resource + */ + public void setUnits(String rUnits) { + if (!UnitsConversionUtil.KNOWN_UNITS.contains(rUnits)) { + throw new IllegalArgumentException( + "Unknown unit '" + units + "'. Known units are " + + UnitsConversionUtil.KNOWN_UNITS); + } + this.units = rUnits; + } + + /** + * Get the resource type. + * + * @return the resource type + */ + public ResourceTypes getResourceType() { + return resourceType; + } + + /** + * Set the resource type. + * + * @param type the resource type + */ + public void setResourceType(ResourceTypes type) { + this.resourceType = type; + } + + /** + * Get the value for the resource. + * + * @return the resource value + */ + public Long getValue() { + return value; + } + + /** + * Set the value for the resource. + * + * @param rValue the resource value + */ + public void setValue(Long rValue) { + this.value = rValue; + } + + /** + * Create a new instance of ResourceInformation from another object. + * + * @param other the object from which the new object should be created + * @return the new ResourceInformation object + */ + public static ResourceInformation newInstance(ResourceInformation other) { + ResourceInformation ret = new ResourceInformation(); + ret.setName(other.getName()); + ret.setResourceType(other.getResourceType()); + ret.setUnits(other.getUnits()); + ret.setValue(other.getValue()); + return ret; + } + + public static ResourceInformation newInstance(String name, String units, + Long value, ResourceTypes type) { + ResourceInformation ret = new ResourceInformation(); + ret.setName(name); + ret.setResourceType(type); + ret.setUnits(units); + ret.setValue(value); + return ret; + } + + public static ResourceInformation newInstance(String name, String units, + Long value) { + return ResourceInformation + .newInstance(name, units, value, ResourceTypes.COUNTABLE); + } + + public static ResourceInformation newInstance(String name, String units) { + return ResourceInformation + .newInstance(name, units, 0L, ResourceTypes.COUNTABLE); + } + + public static ResourceInformation newInstance(String name, Long value) { + return ResourceInformation + .newInstance(name, "", value, ResourceTypes.COUNTABLE); + } + + public static ResourceInformation newInstance(String name) { + return ResourceInformation.newInstance(name, ""); + } + + @Override + public String toString() { + return "name: " + this.name + ", units: " + this.units + ", type: " + + resourceType + ", value: " + value; + } + + public String getShorthandRepresentation() { + return "" + this.value + this.units; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ResourceInformation)) { + return false; + } + ResourceInformation r = (ResourceInformation) obj; + int cmp = + UnitsConversionUtil.compare(this.units, this.value, r.units, r.value); + return this.name.equals(r.getName()) && this.resourceType + .equals(r.getResourceType()) && (cmp == 0); + } + + @Override + public int hashCode() { + final int prime = 263167; + int result = + 939769357 + name.hashCode(); // prime * result = 939769357 initially + result = prime * result + resourceType.hashCode(); + result = prime * result + units.hashCode(); + result = prime * result + value.hashCode(); + return result; + } + + @Override + public int compareTo(ResourceInformation other) { + int diff = this.name.compareTo(other.name); + if (diff == 0) { + diff = UnitsConversionUtil + .compare(this.units, this.value, other.units, other.value); + if (diff == 0) { + diff = this.resourceType.compareTo(other.resourceType); + } + } + return diff; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java new file mode 100644 index 0000000..4277034 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/ResourceNotFoundException.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.exceptions; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This exception is thrown when details of an unknown resource type + * are requested. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class ResourceNotFoundException extends YarnException { + + private static final long serialVersionUID = 10081982L; + + public ResourceNotFoundException(String message) { + super(message); + } + + public ResourceNotFoundException(Throwable cause) { + super(cause); + } + + public ResourceNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java new file mode 100644 index 0000000..7785263 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import java.math.BigInteger; +import java.util.*; + +/** + * A util to convert values in one unit to another. Units refers to whether + * the value is expressed in pico, nano, etc. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class UnitsConversionUtil { + + /** + * Helper class for encapsulating conversion values. + */ + public static class Converter { + private long numerator; + private long denominator; + + Converter(long n, long d) { + this.numerator = n; + this.denominator = d; + } + } + + private static final String[] UNITS = + {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"}; + private static final List<String> SORTED_UNITS = Arrays.asList(UNITS); + public static final Set<String> KNOWN_UNITS = createKnownUnitsSet(); + private static final Converter PICO = + new Converter(1L, 1000L * 1000L * 1000L * 1000L); + private static final Converter NANO = + new Converter(1L, 1000L * 1000L * 1000L); + private static final Converter MICRO = new Converter(1L, 1000L * 1000L); + private static final Converter MILLI = new Converter(1L, 1000L); + private static final Converter BASE = new Converter(1L, 1L); + private static final Converter KILO = new Converter(1000L, 1L); + private static final Converter MEGA = new Converter(1000L * 1000L, 1L); + private static final Converter GIGA = + new Converter(1000L * 1000L * 1000L, 1L); + private static final Converter TERA = + new Converter(1000L * 1000L * 1000L * 1000L, 1L); + private static final Converter PETA = + new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L); + + private static Set<String> createKnownUnitsSet() { + Set<String> ret = new HashSet<>(); + ret.addAll(Arrays.asList(UNITS)); + return ret; + } + + private static Converter getConverter(String unit) { + switch (unit) { + case "p": + return PICO; + case "n": + return NANO; + case "u": + return MICRO; + case "m": + return MILLI; + case "": + return BASE; + case "k": + return KILO; + case "M": + return MEGA; + case "G": + return GIGA; + case "T": + return TERA; + case "P": + return PETA; + default: + throw new IllegalArgumentException( + "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS); + } + } + + /** + * Converts a value from one unit to another. Supported units can be obtained + * by inspecting the KNOWN_UNITS set. + * + * @param fromUnit the unit of the from value + * @param toUnit the target unit + * @param fromValue the value you wish to convert + * @return the value in toUnit + */ + public static Long convert(String fromUnit, String toUnit, Long fromValue) { + if (toUnit == null || fromUnit == null || fromValue == null) { + throw new IllegalArgumentException("One or more arguments are null"); + } + Long tmp; + String overflowMsg = + "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit + + "' will result in an overflow of Long"; + Converter fc = getConverter(fromUnit); + Converter tc = getConverter(toUnit); + Long numerator = fc.numerator * tc.denominator; + Long denominator = fc.denominator * tc.numerator; + if (numerator < denominator) { + if (!toUnit.equals(fromUnit)) { + tmp = Long.MAX_VALUE / numerator; + if (tmp < fromValue) { + throw new IllegalArgumentException(overflowMsg); + } + } + return (fromValue * numerator) / denominator; + } + tmp = numerator / denominator; + if (!toUnit.equals(fromUnit)) { + if ((Long.MAX_VALUE / tmp) < fromValue) { + throw new IllegalArgumentException(overflowMsg); + } + } + return fromValue * tmp; + } + + /** + * Compare a value in a given unit with a value in another unit. The return + * value is equivalent to the value returned by compareTo. + * + * @param unitA first unit + * @param valueA first value + * @param unitB second unit + * @param valueB second value + * @return +1, 0 or -1 depending on whether the relationship is greater than, + * equal to or lesser than + */ + public static int compare(String unitA, Long valueA, String unitB, + Long valueB) { + if (unitA == null || unitB == null || !KNOWN_UNITS.contains(unitA) + || !KNOWN_UNITS.contains(unitB)) { + throw new IllegalArgumentException("Units cannot be null"); + } + if (!KNOWN_UNITS.contains(unitA)) { + throw new IllegalArgumentException("Unknown unit '" + unitA + "'"); + } + if (!KNOWN_UNITS.contains(unitB)) { + throw new IllegalArgumentException("Unknown unit '" + unitB + "'"); + } + Converter unitAC = getConverter(unitA); + Converter unitBC = getConverter(unitB); + if (unitA.equals(unitB)) { + return valueA.compareTo(valueB); + } + int unitAPos = SORTED_UNITS.indexOf(unitA); + int unitBPos = SORTED_UNITS.indexOf(unitB); + try { + Long tmpA = valueA; + Long tmpB = valueB; + if (unitAPos < unitBPos) { + tmpB = convert(unitB, unitA, valueB); + } else { + tmpA = convert(unitA, unitB, valueA); + } + return tmpA.compareTo(tmpB); + } catch (IllegalArgumentException ie) { + BigInteger tmpA = BigInteger.valueOf(valueA); + BigInteger tmpB = BigInteger.valueOf(valueB); + if (unitAPos < unitBPos) { + tmpB = tmpB.multiply(BigInteger.valueOf(unitBC.numerator)); + tmpB = tmpB.multiply(BigInteger.valueOf(unitAC.denominator)); + tmpB = tmpB.divide(BigInteger.valueOf(unitBC.denominator)); + tmpB = tmpB.divide(BigInteger.valueOf(unitAC.numerator)); + } else { + tmpA = tmpA.multiply(BigInteger.valueOf(unitAC.numerator)); + tmpA = tmpA.multiply(BigInteger.valueOf(unitBC.denominator)); + tmpA = tmpA.divide(BigInteger.valueOf(unitAC.denominator)); + tmpA = tmpA.divide(BigInteger.valueOf(unitBC.numerator)); + } + return tmpA.compareTo(tmpB); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5a70298..e40fd4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -53,9 +53,21 @@ message ContainerIdProto { optional int64 id = 3; } +enum ResourceTypesProto { + COUNTABLE = 0; +} + +message ResourceInformationProto { + required string key = 1; + optional int64 value = 2; + optional string units = 3; + optional ResourceTypesProto type = 4; +} + message ResourceProto { optional int64 memory = 1; optional int32 virtual_cores = 2; + repeated ResourceInformationProto resource_value_map = 3; } message ResourceUtilizationProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java new file mode 100644 index 0000000..28f69c9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.conf; + +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.junit.Assert; +import org.junit.Test; + +public class TestResourceInformation { + + @Test + public void testName() { + String name = "yarn.io/test"; + ResourceInformation ri = ResourceInformation.newInstance(name); + Assert.assertEquals("Resource name incorrect", name, ri.getName()); + } + + @Test + public void testUnits() { + String name = "yarn.io/test"; + String units = "m"; + ResourceInformation ri = ResourceInformation.newInstance(name, units); + Assert.assertEquals("Resource name incorrect", name, ri.getName()); + Assert.assertEquals("Resource units incorrect", units, ri.getUnits()); + units = "z"; + try { + ResourceInformation.newInstance(name, units); + Assert.fail(units + "is not a valid unit"); + } catch (IllegalArgumentException ie) { + // do nothing + } + } + + @Test + public void testValue() { + String name = "yarn.io/test"; + Long value = 1l; + ResourceInformation ri = ResourceInformation.newInstance(name, value); + Assert.assertEquals("Resource name incorrect", name, ri.getName()); + Assert.assertEquals("Resource value incorrect", value, ri.getValue()); + } + + @Test + public void testResourceInformation() { + String name = "yarn.io/test"; + Long value = 1l; + String units = "m"; + ResourceInformation ri = + ResourceInformation.newInstance(name, units, value); + Assert.assertEquals("Resource name incorrect", name, ri.getName()); + Assert.assertEquals("Resource value incorrect", value, ri.getValue()); + Assert.assertEquals("Resource units incorrect", units, ri.getUnits()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java new file mode 100644 index 0000000..421768f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import org.junit.Assert; +import org.junit.Test; + +public class TestUnitsConversionUtil { + + @Test + public void testUnitsConversion() { + int value = 5; + String fromUnit = ""; + Long test = Long.valueOf(value); + Assert.assertEquals("pico test failed", + Long.valueOf(value * 1000l * 1000l * 1000l * 1000l), + UnitsConversionUtil.convert(fromUnit, "p", test)); + Assert.assertEquals("nano test failed", + Long.valueOf(value * 1000l * 1000l * 1000l), + UnitsConversionUtil.convert(fromUnit, "n", test)); + Assert + .assertEquals("micro test failed", Long.valueOf(value * 1000l * 1000l), + UnitsConversionUtil.convert(fromUnit, "u", test)); + Assert.assertEquals("milli test failed", Long.valueOf(value * 1000l), + UnitsConversionUtil.convert(fromUnit, "m", test)); + + test = Long.valueOf(value * 1000l * 1000l * 1000l * 1000l * 1000l); + fromUnit = ""; + Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l), + UnitsConversionUtil.convert(fromUnit, "k", test)); + Assert + .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)), + UnitsConversionUtil.convert(fromUnit, "M", test)); + Assert.assertEquals("giga test failed", + Long.valueOf(test / (1000l * 1000l * 1000l)), + UnitsConversionUtil.convert(fromUnit, "G", test)); + Assert.assertEquals("tera test failed", + Long.valueOf(test / (1000l * 1000l * 1000l * 1000l)), + UnitsConversionUtil.convert(fromUnit, "T", test)); + Assert.assertEquals("peta test failed", + Long.valueOf(test / (1000l * 1000l * 1000l * 1000l * 1000l)), + UnitsConversionUtil.convert(fromUnit, "P", test)); + + Assert.assertEquals("nano to pico test failed", Long.valueOf(value * 1000l), + UnitsConversionUtil.convert("n", "p", Long.valueOf(value))); + + Assert.assertEquals("mega to giga test failed", Long.valueOf(value), + UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l))); + } + + @Test + public void testOverflow() { + Long test = Long.valueOf(5 * 1000l * 1000l * 1000l * 1000l * 1000l); + try { + UnitsConversionUtil.convert("P", "p", test); + Assert.fail("this operation should result in an overflow"); + } catch (IllegalArgumentException ie) { + ; // do nothing + } + try { + UnitsConversionUtil.convert("m", "p", Long.MAX_VALUE - 1); + Assert.fail("this operation should result in an overflow"); + } catch (IllegalArgumentException ie) { + ; // do nothing + } + } + + @Test + public void testCompare() { + String unitA = "P"; + Long valueA = Long.valueOf(1); + String unitB = "p"; + Long valueB = Long.valueOf(2); + Assert.assertEquals(1, + UnitsConversionUtil.compare(unitA, valueA, unitB, valueB)); + Assert.assertEquals(-1, + UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); + Assert.assertEquals(0, + UnitsConversionUtil.compare(unitA, valueA, unitA, valueA)); + Assert.assertEquals(-1, + UnitsConversionUtil.compare(unitA, valueA, unitA, valueB)); + Assert.assertEquals(1, + UnitsConversionUtil.compare(unitA, valueB, unitA, valueA)); + + unitB = "T"; + Assert.assertEquals(1, + UnitsConversionUtil.compare(unitA, valueA, unitB, valueB)); + Assert.assertEquals(-1, + UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); + Assert.assertEquals(0, + UnitsConversionUtil.compare(unitA, valueA, unitB, 1000l)); + + unitA = "p"; + unitB = "n"; + Assert.assertEquals(-1, + UnitsConversionUtil.compare(unitA, valueA, unitB, valueB)); + Assert.assertEquals(1, + UnitsConversionUtil.compare(unitB, valueB, unitA, valueA)); + Assert.assertEquals(0, + UnitsConversionUtil.compare(unitA, 1000l, unitB, valueA)); + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index ab283e7..760f809 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.AMCommand; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -71,6 +72,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypesProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateTypeProto; import org.apache.hadoop.yarn.server.api.ContainerType; @@ -431,6 +433,17 @@ public class ProtoUtils { convertToProtoFormat(UpdateContainerError t) { return ((UpdateContainerErrorPBImpl) t).getProto(); } + + /* + * ResourceTypes + */ + public static ResourceTypesProto converToProtoFormat(ResourceTypes e) { + return ResourceTypesProto.valueOf(e.name()); + } + + public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { + return ResourceTypes.valueOf(e.name()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 6686696..14e5090 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -21,9 +21,17 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; + +import java.util.*; @Private @Unstable @@ -31,7 +39,9 @@ public class ResourcePBImpl extends Resource { ResourceProto proto = ResourceProto.getDefaultInstance(); ResourceProto.Builder builder = null; boolean viaProto = false; - + + private Map<String, ResourceInformation> resources; + public ResourcePBImpl() { builder = ResourceProto.newBuilder(); } @@ -39,9 +49,12 @@ public class ResourcePBImpl extends Resource { public ResourcePBImpl(ResourceProto proto) { this.proto = proto; viaProto = true; + this.resources = null; + initResources(); } - + public ResourceProto getProto() { + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; @@ -57,13 +70,21 @@ public class ResourcePBImpl extends Resource { @Override @SuppressWarnings("deprecation") public int getMemory() { - return (int) getMemorySize(); + return (int) this.getMemorySize(); } @Override public long getMemorySize() { - ResourceProtoOrBuilder p = viaProto ? proto : builder; - return p.getMemory(); + try { + ResourceInformation ri = + this.getResourceInformation(ResourceInformation.MEMORY.getName()); + return (int) UnitsConversionUtil + .convert(ri.getUnits(), "M", ri.getValue()).longValue(); + } catch (YarnException ye) { + // memory should always be present + initResourcesMap(); + return 0; + } } @Override @@ -74,30 +95,172 @@ public class ResourcePBImpl extends Resource { @Override public void setMemorySize(long memory) { - maybeInitBuilder(); - builder.setMemory(memory); + setResourceInformation(ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits(), memory)); + } @Override public int getVirtualCores() { - ResourceProtoOrBuilder p = viaProto ? proto : builder; - return p.getVirtualCores(); + try { + return (int) this.getResourceValue(ResourceInformation.VCORES.getName()) + .longValue(); + } catch (YarnException ye) { + // vcores should always be present + initResourcesMap(); + return 0; + } } @Override public void setVirtualCores(int vCores) { + try { + setResourceValue(ResourceInformation.VCORES.getName(), + Long.valueOf(vCores)); + } catch (ResourceNotFoundException re) { + this.setResourceInformation(ResourceInformation.VCORES.getName(), + ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), + (long) vCores)); + } + } + + private void initResources() { + if (this.resources != null) { + return; + } + ResourceProtoOrBuilder p = viaProto ? proto : builder; + initResourcesMap(); + for (ResourceInformationProto entry : p.getResourceValueMapList()) { + ResourceTypes type = + entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : + ResourceTypes.COUNTABLE; + String units = entry.hasUnits() ? entry.getUnits() : ""; + Long value = entry.hasValue() ? entry.getValue() : 0L; + ResourceInformation ri = + ResourceInformation.newInstance(entry.getKey(), units, value, type); + resources.put(ri.getName(), ri); + } + if(this.getMemory() != p.getMemory()) { + setMemorySize(p.getMemory()); + } + if(this.getVirtualCores() != p.getVirtualCores()) { + setVirtualCores(p.getVirtualCores()); + } + } + + @Override + public void setResources(Map<String, ResourceInformation> resources) { maybeInitBuilder(); - builder.setVirtualCores(vCores); + if (resources == null || resources.isEmpty()) { + builder.clearResourceValueMap(); + } else { + for (Map.Entry<String, ResourceInformation> entry : resources.entrySet()) { + if (!entry.getKey().equals(entry.getValue().getName())) { + entry.getValue().setName(entry.getKey()); + } + } + } + this.resources = resources; } @Override - public int compareTo(Resource other) { - long diff = this.getMemorySize() - other.getMemorySize(); - if (diff == 0) { - diff = this.getVirtualCores() - other.getVirtualCores(); + public void setResourceInformation(String resource, + ResourceInformation resourceInformation) { + maybeInitBuilder(); + if (resource == null || resourceInformation == null) { + throw new IllegalArgumentException( + "resource and/or resourceInformation cannot be null"); + } + if (!resource.equals(resourceInformation.getName())) { + resourceInformation.setName(resource); } - return diff == 0 ? 0 : (diff > 0 ? 1 : -1); + initResourcesMap(); + resources.put(resource, resourceInformation); + } + + @Override + public void setResourceValue(String resource, Long value) + throws ResourceNotFoundException { + maybeInitBuilder(); + if (resource == null) { + throw new IllegalArgumentException("resource type object cannot be null"); + } + if (resources == null || (!resources.containsKey(resource))) { + throw new ResourceNotFoundException( + "Resource " + resource + " not found"); + } + ResourceInformation ri = resources.get(resource); + ri.setValue(value); + resources.put(resource, ri); + } + + @Override + public Map<String, ResourceInformation> getResources() { + initResources(); + return Collections.unmodifiableMap(this.resources); + } + + @Override + public ResourceInformation getResourceInformation(String resource) + throws YarnException { + initResources(); + if (this.resources.containsKey(resource)) { + return this.resources.get(resource); + } + throw new ResourceNotFoundException("Could not find entry for " + resource); + } + + @Override + public Long getResourceValue(String resource) throws YarnException { + initResources(); + if (this.resources.containsKey(resource)) { + return this.resources.get(resource).getValue(); + } + throw new ResourceNotFoundException("Could not find entry for " + resource); + } + + private void initResourcesMap() { + if (resources == null) { + resources = new HashMap<>(); + } + ResourceInformation ri; + if (!resources.containsKey(ResourceInformation.MEMORY.getName())) { + ri = ResourceInformation + .newInstance(ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits()); + this.resources.put(ResourceInformation.MEMORY.getName(), ri); + } + if (!resources.containsKey(ResourceInformation.VCORES.getName())) { + ri = + ResourceInformation.newInstance(ResourceInformation.VCORES.getName()); + this.resources.put(ResourceInformation.VCORES.getName(), ri); + } + } + + synchronized private void mergeLocalToBuilder() { + builder.clearResourceValueMap(); + if (resources != null && !resources.isEmpty()) { + for (Map.Entry<String, ResourceInformation> entry : resources.entrySet()) { + ResourceInformationProto.Builder e = ResourceInformationProto.newBuilder(); + e.setKey(entry.getKey()); + e.setUnits(entry.getValue().getUnits()); + e.setType( + ProtoUtils.converToProtoFormat(entry.getValue().getResourceType())); + e.setValue(entry.getValue().getValue()); + builder.addResourceValueMap(e); + } + } + builder.setMemory(this.getMemory()); + builder.setVirtualCores(this.getVirtualCores()); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; } - - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 462e02a..59ee009 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -21,102 +21,137 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; -@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"}) +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) @Unstable public class Resources { - - // Java doesn't have const :( - private static final Resource NONE = new Resource() { + + /** + * Helper class to create a resource with a fixed value for all resource + * types. For example, a NONE resource which returns 0 for any resource type. + */ + static class FixedValueResource extends Resource { + + private Map<String, ResourceInformation> resources; + private Long resourceValue; + private String name; + + /** + * Constructor for a fixed value resource + * @param rName the name of the resource + * @param value the fixed value to be returned for all resource types + */ + FixedValueResource(String rName, Long value) { + this.resourceValue = value; + this.name = rName; + resources = initResourceMap(); + } + + private int resourceValueToInt() { + if(this.resourceValue > Integer.MAX_VALUE) { + return Integer.MAX_VALUE; + } + return this.resourceValue.intValue(); + } @Override @SuppressWarnings("deprecation") public int getMemory() { - return 0; + return resourceValueToInt(); } @Override public long getMemorySize() { - return 0; + return this.resourceValue; } @Override - public void setMemorySize(long memory) { - throw new RuntimeException("NONE cannot be modified!"); + public void setMemory(int memory) { + throw new RuntimeException(name + " cannot be modified!"); } @Override @SuppressWarnings("deprecation") - public void setMemory(int memory) { - throw new RuntimeException("NONE cannot be modified!"); + public void setMemorySize(long memory) { + throw new RuntimeException(name + " cannot be modified!"); } @Override public int getVirtualCores() { - return 0; + return resourceValueToInt(); } @Override - public void setVirtualCores(int cores) { - throw new RuntimeException("NONE cannot be modified!"); + public void setVirtualCores(int virtualCores) { + throw new RuntimeException(name + " cannot be modified!"); } @Override - public int compareTo(Resource o) { - long diff = 0 - o.getMemorySize(); - if (diff == 0) { - diff = 0 - o.getVirtualCores(); - } - return Long.signum(diff); + public Map<String, ResourceInformation> getResources() { + return Collections.unmodifiableMap(this.resources); } - - }; - - private static final Resource UNBOUNDED = new Resource() { @Override - @SuppressWarnings("deprecation") - public int getMemory() { - return Integer.MAX_VALUE; + public ResourceInformation getResourceInformation(String resource) + throws YarnException { + if (resources.containsKey(resource)) { + ResourceInformation value = this.resources.get(resource); + ResourceInformation ret = ResourceInformation.newInstance(value); + ret.setValue(resourceValue); + return ret; + } + throw new YarnException("" + resource + " not found"); } @Override - public long getMemorySize() { - return Long.MAX_VALUE; + public Long getResourceValue(String resource) throws YarnException { + if (resources.containsKey(resource)) { + return resourceValue; + } + throw new YarnException("" + resource + " not found"); } @Override - @SuppressWarnings("deprecation") - public void setMemory(int memory) { - throw new RuntimeException("UNBOUNDED cannot be modified!"); + public void setResources(Map<String, ResourceInformation> resources) { + throw new RuntimeException(name + " cannot be modified!"); } @Override - public void setMemorySize(long memory) { - throw new RuntimeException("UNBOUNDED cannot be modified!"); + public void setResourceInformation(String resource, + ResourceInformation resourceInformation) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); } @Override - public int getVirtualCores() { - return Integer.MAX_VALUE; + public void setResourceValue(String resource, Long value) + throws ResourceNotFoundException { + throw new RuntimeException(name + " cannot be modified!"); } - @Override - public void setVirtualCores(int cores) { - throw new RuntimeException("UNBOUNDED cannot be modified!"); + private Map<String, ResourceInformation> initResourceMap() { + Map<String, ResourceInformation> tmp = new HashMap<>(); + // Due to backwards compat, the max value for memory and vcores + // needs to be Integer.MAX_VALUE + int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE : + resourceValue.intValue(); + tmp.put(ResourceInformation.MEMORY.getName(), ResourceInformation + .newInstance(ResourceInformation.MEMORY.getName(), + ResourceInformation.MEMORY_MB.getUnits(), (long) max)); + tmp.put(ResourceInformation.VCORES.getName(), ResourceInformation + .newInstance(ResourceInformation.VCORES.getName(), (long) max)); + return tmp; } - @Override - public int compareTo(Resource o) { - long diff = Long.MAX_VALUE - o.getMemorySize(); - if (diff == 0) { - diff = Integer.MAX_VALUE - o.getVirtualCores(); - } - return Long.signum(diff); - } - - }; + } public static Resource createResource(int memory) { return createResource(memory, (memory > 0) ? 1 : 0); @@ -129,6 +164,11 @@ public class Resources { return resource; } + private static final Resource UNBOUNDED = + new FixedValueResource("UNBOUNDED", Long.MAX_VALUE); + + private static final Resource NONE = new FixedValueResource("NONE", 0L); + public static Resource createResource(long memory) { return createResource(memory, (memory > 0) ? 1 : 0); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d322313/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 10323d5..dc57092 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -134,6 +134,7 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.ResourceAllocationRequest; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -341,6 +342,8 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { typeValueCache.put(SerializedException.class, SerializedException.newInstance(new IOException("exception for test"))); generateByNewInstance(ExecutionTypeRequest.class); + typeValueCache.put(ResourceInformation.class, ResourceInformation + .newInstance("localhost.test/sample", 1l)); generateByNewInstance(LogAggregationContext.class); generateByNewInstance(ApplicationId.class); generateByNewInstance(ApplicationAttemptId.class); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
