svn commit: r916290 - in /hadoop/common/trunk: ./ .eclipse.templates/ ivy/ src/java/org/apache/hadoop/security/token/ src/test/core/org/apache/hadoop/io/compress/ src/test/core/org/apache/hadoop/secur
Author: omalley Date: Thu Feb 25 14:14:18 2010 New Revision: 916290 URL: http://svn.apache.org/viewvc?rev=916290&view=rev Log: HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from a url-safe string and change the commons-code library to 1.4. (omalley) Modified: hadoop/common/trunk/.eclipse.templates/.classpath hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/ivy/hadoop-core-template.xml hadoop/common/trunk/ivy/libraries.properties hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java hadoop/common/trunk/src/test/core/org/apache/hadoop/io/compress/TestCodec.java hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/TestToken.java Modified: hadoop/common/trunk/.eclipse.templates/.classpath URL: http://svn.apache.org/viewvc/hadoop/common/trunk/.eclipse.templates/.classpath?rev=916290&r1=916289&r2=916290&view=diff == --- hadoop/common/trunk/.eclipse.templates/.classpath (original) +++ hadoop/common/trunk/.eclipse.templates/.classpath Thu Feb 25 14:14:18 2010 @@ -7,7 +7,7 @@ - + Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916290&r1=916289&r2=916290&view=diff == --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Thu Feb 25 14:14:18 2010 @@ -161,6 +161,9 @@ HADOOP-6543. Allows secure clients to talk to unsecure clusters. (Kan Zhang via ddas) +HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from +a url-safe string and change the commons-code library to 1.4. (omalley) + OPTIMIZATIONS HADOOP-6467. Improve the performance on HarFileSystem.listStatus(..). Modified: hadoop/common/trunk/ivy/hadoop-core-template.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/ivy/hadoop-core-template.xml?rev=916290&r1=916289&r2=916290&view=diff == --- hadoop/common/trunk/ivy/hadoop-core-template.xml (original) +++ hadoop/common/trunk/ivy/hadoop-core-template.xml Thu Feb 25 14:14:18 2010 @@ -41,7 +41,7 @@ commons-codec commons-codec - 1.3 + 1.4 commons-net Modified: hadoop/common/trunk/ivy/libraries.properties URL: http://svn.apache.org/viewvc/hadoop/common/trunk/ivy/libraries.properties?rev=916290&r1=916289&r2=916290&view=diff == --- hadoop/common/trunk/ivy/libraries.properties (original) +++ hadoop/common/trunk/ivy/libraries.properties Thu Feb 25 14:14:18 2010 @@ -23,7 +23,7 @@ commons-cli.version=1.2 commons-cli2.version=2.0-mahout -commons-codec.version=1.3 +commons-codec.version=1.4 commons-collections.version=3.1 commons-httpclient.version=3.0.1 commons-lang.version=2.4 Modified: hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java?rev=916290&r1=916289&r2=916290&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/security/token/Token.java Thu Feb 25 14:14:18 2010 @@ -21,9 +21,15 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Arrays; +import org.apache.commons.codec.binary.Base64; + +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; /** @@ -47,7 +53,21 @@ kind = id.getKind(); service = new Text(); } - + + /** + * Construct a token from the components. + * @param identifier the token identifier + * @param password the token's password + * @param kind the kind of token + * @param service the service for this token + */ + public Token(byte[] identifier, byte[] password, Text kind, Text service) { +this.identifier = identifier; +this.password = password; +this.kind = kind; +this.service = service; + } + /** * Default constructor */ @@ -123,4 +143,103 @@ kind.write(out); service.write(out); } + + /** + * Generate a string with the url-quoted base64 encoded serialized form + * of the Writable. + * @param obj the object to serialize + * @return the encoded string + * @throws IOException + */ + private static String encodeWritable(Writable obj) throws IOException { +DataOutputBuffer buf = new DataOutputBuffer(); +obj.write
svn commit: r916390 - in /hadoop/common/trunk: CHANGES.txt src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
Author: omalley Date: Thu Feb 25 18:34:05 2010 New Revision: 916390 URL: http://svn.apache.org/viewvc?rev=916390&view=rev Log: HADOOP-6596. Add a version field to the AbstractDelegationTokenIdentifier's serialized value. (omalley) Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916390&r1=916389&r2=916390&view=diff == --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Thu Feb 25 18:34:05 2010 @@ -164,6 +164,9 @@ HADOOP-6579. Provide a mechanism for encoding/decoding Tokens from a url-safe string and change the commons-code library to 1.4. (omalley) +HADOOP-6596. Add a version field to the AbstractDelegationTokenIdentifier's +serialized value. (omalley) + OPTIMIZATIONS HADOOP-6467. Improve the performance on HarFileSystem.listStatus(..). Modified: hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java?rev=916390&r1=916389&r2=916390&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java Thu Feb 25 18:34:05 2010 @@ -34,6 +34,7 @@ @InterfaceAudience.LimitedPrivate({HDFS, MAPREDUCE}) public abstract class AbstractDelegationTokenIdentifier extends TokenIdentifier { + private static final byte VERSION = 0; private Text owner; private Text renewer; @@ -145,6 +146,11 @@ } public void readFields(DataInput in) throws IOException { +byte version = in.readByte(); +if (version != VERSION) { + throw new IOException("Unknown version of delegation token " + + version); +} owner.readFields(in); renewer.readFields(in); realUser.readFields(in); @@ -155,6 +161,7 @@ } public void write(DataOutput out) throws IOException { +out.writeByte(VERSION); owner.write(out); renewer.write(out); realUser.write(out);
[Hadoop Wiki] Update of "PoweredBy" by voyager
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "PoweredBy" page has been changed by voyager. http://wiki.apache.org/hadoop/PoweredBy?action=diff&rev1=178&rev2=179 -- * [[http://www.weblab.infosci.cornell.edu/|Cornell University Web Lab]] * Generating web graphs on 100 nodes (dual 2.4GHz Xeon Processor, 2 GB RAM, 72GB Hard Drive) - - - * [[http://www.deepdyve.com|Deepdyve]] * Elastic cluster with 5-80 nodes * We use hadoop to create our indexes of deep web content and to provide a high availability and high bandwidth storage service for index shards for our search cluster. @@ -103, +100 @@ * We use Hadoop to filter and index our listings, removing exact duplicates and grouping similar ones. * We plan to use Pig very shortly to produce statistics. - * [[http://blog.espol.edu.ec/hadoop/|ESPOL University (Escuela Superior Politécnica del Litoral) in Guayaquil, Ecuador]] * 4 nodes proof-of-concept cluster. * We use Hadoop in a Data-Intensive Computing capstone course. The course projects cover topics like information retrieval, machine learning, social network analysis, business intelligence, and network security. @@ -117, +113 @@ * Facial similarity and recognition across large datasets. * Image content based advertising and auto-tagging for social media. * Image based video copyright protection. - * [[http://www.facebook.com/|Facebook]] * We use Hadoop to store copies of internal log and dimension data sources and use it as a source for reporting/analytics and machine learning. @@ -141, +136 @@ * [[http://www.google.com|Google]] * [[http://www.google.com/intl/en/press/pressrel/20071008_ibm_univ.html|University Initiative to Address Internet-Scale Computing Challenges]] - - * [[http://www.gruter.com|Gruter. Corp.]] * 30 machine cluster (4 cores, 1TB~2TB/machine storage) * storage for blog data and web documents @@ -226, +219 @@ * [[http://www.lotame.com|Lotame]] * Using Hadoop and Hbase for storage, log analysis, and pattern discovery/analysis. + + + * [[http://www.makara.com//|Makara]] + * Using ZooKeeper on 2 node cluster on VMware workstation, Amazon EC2, Zen + * Using zkpython + * Looking into expanding into 100 node cluster + * [[http://www.crmcs.com//|MicroCode]] * 18 node cluster (Quad-Core Intel Xeon, 1TB/node storage)
svn commit: r916467 - in /hadoop/common/trunk: ./ src/java/ src/java/org/apache/hadoop/conf/ src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/http/ src/java/org/apache/hadoop/ipc/ src/java/or
Author: ddas Date: Thu Feb 25 21:39:38 2010 New Revision: 916467 URL: http://svn.apache.org/viewvc?rev=916467&view=rev Log: HADOOP-6568. Adds authorization for the default servlets. Contributed by Vinod Kumar Vavilapalli. Added: hadoop/common/trunk/src/java/org/apache/hadoop/http/AdminAuthorizedServlet.java Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/java/core-default.xml hadoop/common/trunk/src/java/org/apache/hadoop/conf/ConfServlet.java hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java hadoop/common/trunk/src/java/org/apache/hadoop/log/LogLevel.java hadoop/common/trunk/src/java/org/apache/hadoop/metrics/MetricsServlet.java hadoop/common/trunk/src/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/CLITestHelper.java hadoop/common/trunk/src/test/core/org/apache/hadoop/http/TestHttpServer.java hadoop/common/trunk/src/test/core/org/apache/hadoop/ipc/TestRPC.java Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916467&r1=916466&r2=916467&view=diff == --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Thu Feb 25 21:39:38 2010 @@ -62,6 +62,9 @@ to change the default 1 MB, the maximum size when large IPC handler response buffer is reset. (suresh) +HADOOP-6568. Adds authorization for the default servlets. +(Vinod Kumar Vavilapalli via ddas) + IMPROVEMENTS HADOOP-6283. Improve the exception messages thrown by Modified: hadoop/common/trunk/src/java/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/core-default.xml?rev=916467&r1=916466&r2=916467&view=diff == --- hadoop/common/trunk/src/java/core-default.xml (original) +++ hadoop/common/trunk/src/java/core-default.xml Thu Feb 25 21:39:38 2010 @@ -54,6 +54,16 @@ + hadoop.cluster.administrators + Users and/or groups who are designated as the administrators of a + hadoop cluster. For specifying a list of users and groups the format to use + is "user1,user2 group1,group". If set to '*', it allows all users/groups to + do administrations operations of the cluster. If set to '', it allows none. + + ${user.name} + + + hadoop.security.authorization false Is service-level authorization enabled? Modified: hadoop/common/trunk/src/java/org/apache/hadoop/conf/ConfServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/conf/ConfServlet.java?rev=916467&r1=916466&r2=916467&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/conf/ConfServlet.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/conf/ConfServlet.java Thu Feb 25 21:39:38 2010 @@ -52,6 +52,13 @@ @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + +// Do the authorization +if (!HttpServer.hasAdministratorAccess(getServletContext(), request, +response)) { + return; +} + String format = request.getParameter(FORMAT_PARAM); if (null == format) { format = FORMAT_XML; Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=916467&r1=916466&r2=916467&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Thu Feb 25 21:39:38 2010 @@ -133,5 +133,12 @@ public static final String HADOOP_SECURITY_GROUP_MAPPING = "hadoop.security.group.mapping"; public static final String HADOOP_SECURITY_GROUPS_CACHE_SECS = "hadoop.security.groups.cache.secs"; public static final String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; + public static final String HADOOP_SECURITY_AUTHORIZATION = + "hadoop.security.authorization"; + /** + * ACL denoting the administrator ACLs for a hadoop cluster. + */ + public final static String HADOOP_CLUSTER_ADMINISTRATORS_PROPERTY = + "hadoop.cluster.administrators"; } Added: hadoop/common/trunk/src/java/org/apache/hadoop/http/AdminAuthorizedServlet.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/http/AdminAuthorizedServlet.java?rev=916467&view=auto
svn commit: r916468 - in /hadoop/common/trunk: CHANGES.txt src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java src/test/core/org/apache/hadoop/security/token
Author: shv Date: Thu Feb 25 21:43:20 2010 New Revision: 916468 URL: http://svn.apache.org/viewvc?rev=916468&view=rev Log: HADOOP-6573. Support for persistent delegation tokens. Contributed by Jitendra Pandey. Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java hadoop/common/trunk/src/test/core/org/apache/hadoop/security/token/delegation/TestDelegationToken.java Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916468&r1=916467&r2=916468&view=diff == --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Thu Feb 25 21:43:20 2010 @@ -170,6 +170,9 @@ HADOOP-6596. Add a version field to the AbstractDelegationTokenIdentifier's serialized value. (omalley) +HADOOP-6573. Support for persistent delegation tokens. +(Jitendra Pandey via shv) + OPTIMIZATIONS HADOOP-6467. Improve the performance on HarFileSystem.listStatus(..). Modified: hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java?rev=916468&r1=916467&r2=916468&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java Thu Feb 25 21:43:20 2010 @@ -52,23 +52,30 @@ /** * Cache of currently valid tokens, mapping from DelegationTokenIdentifier - * to DelegationTokenInformation. Protected by its own lock. + * to DelegationTokenInformation. Protected by this object lock. */ - private final Map currentTokens + protected final Map currentTokens = new HashMap(); /** - * Sequence number to create DelegationTokenIdentifier + * Sequence number to create DelegationTokenIdentifier. + * Protected by this object lock. */ - private int delegationTokenSequenceNumber = 0; + protected int delegationTokenSequenceNumber = 0; - private final Map allKeys + /** + * Access to allKeys is protected by this object lock + */ + protected final Map allKeys = new HashMap(); /** - * Access to currentId and currentKey is protected by this object lock. + * Access to currentId is protected by this object lock. + */ + protected int currentId = 0; + /** + * Access to currentKey is protected by this object lock */ - private int currentId = 0; private DelegationKey currentKey; private long keyUpdateInterval; @@ -76,7 +83,7 @@ private long tokenRemoverScanInterval; private long tokenRenewInterval; private Thread tokenRemoverThread; - private volatile boolean running; + protected volatile boolean running; public AbstractDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenMaxLifetime, long delegationTokenRenewInterval, @@ -112,27 +119,50 @@ return allKeys.values().toArray(new DelegationKey[0]); } - /** Update the current master key */ - private synchronized void updateCurrentKey() throws IOException { + protected void logUpdateMasterKey(DelegationKey key) throws IOException { +return; + } + + /** + * Update the current master key + * This is called once by startThreads before tokenRemoverThread is created, + * and only by tokenRemoverThread afterwards. + */ + private void updateCurrentKey() throws IOException { LOG.info("Updating the current master key for generating delegation tokens"); /* Create a new currentKey with an estimated expiry date. */ -currentId++; -currentKey = new DelegationKey(currentId, System.currentTimeMillis() +int newCurrentId; +synchronized (this) { + newCurrentId = currentId+1; +} +DelegationKey newKey = new DelegationKey(newCurrentId, System +.currentTimeMillis() + keyUpdateInterval + tokenMaxLifetime, generateSecret()); -allKeys.put(currentKey.getKeyId(), currentKey); +//Log must be invoked outside the lock on 'this' +logUpdateMasterKey(newKey); +synchronized (this) { + currentId = newKey.getKeyId(); + currentKey = newKey; + allKeys.put(currentKey.getKeyId(), currentKey); +} } - /** Update the current master key for generating delegation tokens */ - public synchronized void rollMasterKey() throws IOException { -removeExpiredKeys(); -/* set final expiry date for retiring currentKey */ -currentKey.setExpiryDate(System.currentTimeMillis() + tokenMaxLifetime); -/* - * current
svn commit: r916529 - in /hadoop/common/trunk: CHANGES.txt src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java src/java/org/apache/hadoop/ipc/Server.java
Author: shv Date: Fri Feb 26 01:37:57 2010 New Revision: 916529 URL: http://svn.apache.org/viewvc?rev=916529&view=rev Log: HADOOP-1849. Add undocumented configuration parameter for per handler call queue size in IPC Server. Contributed by Konstantin Shvachko. Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=916529&r1=916528&r2=916529&view=diff == --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Fri Feb 26 01:37:57 2010 @@ -1392,14 +1392,6 @@ HADOOP-6218. Adds a feature where TFile can be split by Record Sequence number. (Hong Tang and Raghu Angadi via ddas) - IMPROVEMENTS - -HADOOP-5611. Fix C++ libraries to build on Debian Lenny. (Todd Lipcon -via tomwhite) - -HADOOP-5612. Some c++ scripts are not chmodded before ant execution. -(Todd Lipcon via tomwhite) - BUG FIXES HADOOP-6231. Allow caching of filesystem instances to be disabled on a @@ -1424,6 +1416,17 @@ HADOOP-6498. IPC client bug may cause rpc call hang. (Ruyue Ma and hairong via hairong) + IMPROVEMENTS + +HADOOP-5611. Fix C++ libraries to build on Debian Lenny. (Todd Lipcon +via tomwhite) + +HADOOP-5612. Some c++ scripts are not chmodded before ant execution. +(Todd Lipcon via tomwhite) + +HADOOP-1849. Add undocumented configuration parameter for per handler +call queue size in IPC Server. (shv) + Release 0.20.1 - 2009-09-01 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=916529&r1=916528&r2=916529&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Fri Feb 26 01:37:57 2010 @@ -123,6 +123,15 @@ "ipc.server.max.response.size"; public static final int IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT = 1024*1024; + /** + * How many calls per handler are allowed in the queue. + */ + public static final String IPC_SERVER_HANDLER_QUEUE_SIZE_KEY = + "ipc.server.handler.queue.size"; + /** + * The default number of calls per handler in the queue. + */ + public static final int IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT = 100; public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY = "hadoop.rpc.socket.factory.class.default"; Modified: hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java?rev=916529&r1=916528&r2=916529&view=diff == --- hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java Fri Feb 26 01:37:57 2010 @@ -60,8 +60,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; - -import static org.apache.hadoop.fs.CommonConfigurationKeys.*; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.ipc.metrics.RpcMetrics; @@ -98,12 +96,7 @@ // 3 : Introduce the protocol into the RPC connection header // 4 : Introduced SASL security layer public static final byte CURRENT_VERSION = 4; - - /** - * How many calls/handler are allowed in the queue. - */ - private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100; - + /** * Initial and max size of response buffer */ @@ -1288,9 +1281,12 @@ this.paramClass = paramClass; this.handlerCount = handlerCount; this.socketSendBufferSize = 0; -this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER; -this.maxRespSize = conf.getInt(IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY, - IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT); +this.maxQueueSize = handlerCount * conf.getInt( +CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, +CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT); +this.maxRespSize = conf.getInt( +CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY, +CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT); thi
svn commit: r916530 - in /hadoop/common/branches/branch-0.21: CHANGES.txt src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java src/java/org/apache/hadoop/ipc/Server.java
Author: shv Date: Fri Feb 26 01:45:38 2010 New Revision: 916530 URL: http://svn.apache.org/viewvc?rev=916530&view=rev Log: HADOOP-1849. Merge -r 916528:916529 from trunk to branch-0.21. Modified: hadoop/common/branches/branch-0.21/CHANGES.txt hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java Modified: hadoop/common/branches/branch-0.21/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/CHANGES.txt?rev=916530&r1=916529&r2=916530&view=diff == --- hadoop/common/branches/branch-0.21/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.21/CHANGES.txt Fri Feb 26 01:45:38 2010 @@ -1185,6 +1185,11 @@ HADOOP-6498. IPC client bug may cause rpc call hang. (Ruyue Ma and hairong via hairong) + IMPROVEMENTS + +HADOOP-1849. Add undocumented configuration parameter for per handler +call queue size in IPC Server. (shv) + Release 0.20.1 - 2009-09-01 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=916530&r1=916529&r2=916530&view=diff == --- hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Fri Feb 26 01:45:38 2010 @@ -119,6 +119,16 @@ public static final int IPC_CLIENT_IDLETHRESHOLD_DEFAULT = 4000; public static final String IPC_SERVER_TCPNODELAY_KEY = "ipc.server.tcpnodelay"; public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = false; + /** + * How many calls per handler are allowed in the queue. + */ + public static final String IPC_SERVER_HANDLER_QUEUE_SIZE_KEY = + "ipc.server.handler.queue.size"; + /** + * The default number of calls per handler in the queue. + */ + public static final int IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT = 100; + public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY = "hadoop.rpc.socket.factory.class.default"; Modified: hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java?rev=916530&r1=916529&r2=916530&view=diff == --- hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/branch-0.21/src/java/org/apache/hadoop/ipc/Server.java Fri Feb 26 01:45:38 2010 @@ -60,6 +60,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; @@ -84,12 +85,7 @@ // 1 : Introduce ping and server does not throw away RPCs // 3 : Introduce the protocol into the RPC connection header public static final byte CURRENT_VERSION = 3; - - /** - * How many calls/handler are allowed in the queue. - */ - private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100; - + /** * Initial and max size of response buffer */ @@ -1034,7 +1030,9 @@ this.paramClass = paramClass; this.handlerCount = handlerCount; this.socketSendBufferSize = 0; -this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER; +this.maxQueueSize = handlerCount * conf.getInt( +CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, +CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT); this.callQueue = new LinkedBlockingQueue(maxQueueSize); this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000); this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
svn commit: r916531 - in /hadoop/common/branches/branch-0.20: CHANGES.txt src/core/org/apache/hadoop/ipc/Server.java
Author: shv Date: Fri Feb 26 01:46:19 2010 New Revision: 916531 URL: http://svn.apache.org/viewvc?rev=916531&view=rev Log: HADOOP-1849. Merge -r 916528:916529 from trunk to branch-0.20. Modified: hadoop/common/branches/branch-0.20/CHANGES.txt hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/ipc/Server.java Modified: hadoop/common/branches/branch-0.20/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/CHANGES.txt?rev=916531&r1=916530&r2=916531&view=diff == --- hadoop/common/branches/branch-0.20/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.20/CHANGES.txt Fri Feb 26 01:46:19 2010 @@ -126,6 +126,9 @@ HADOOP-5612. Some c++ scripts are not chmodded before ant execution. (Todd Lipcon via tomwhite) +HADOOP-1849. Add undocumented configuration parameter for per handler +call queue size in IPC Server. (shv) + Release 0.20.1 - 2009-09-01 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/ipc/Server.java?rev=916531&r1=916530&r2=916531&view=diff == --- hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/branch-0.20/src/core/org/apache/hadoop/ipc/Server.java Fri Feb 26 01:46:19 2010 @@ -88,7 +88,9 @@ /** * How many calls/handler are allowed in the queue. */ - private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100; + private static final int IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT = 100; + private static final String IPC_SERVER_HANDLER_QUEUE_SIZE_KEY = +"ipc.server.handler.queue.size"; public static final Log LOG = LogFactory.getLog(Server.class); @@ -1016,7 +1018,9 @@ this.paramClass = paramClass; this.handlerCount = handlerCount; this.socketSendBufferSize = 0; -this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER; +this.maxQueueSize = handlerCount * conf.getInt( +IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, +IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT); this.callQueue = new LinkedBlockingQueue(maxQueueSize); this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000); this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
[Hadoop Wiki] Update of "Hive/PoweredBy" by ZhengShao
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "Hive/PoweredBy" page has been changed by ZhengShao. The comment on this change is: Adding Tao Bao per request from Mafish Liu. http://wiki.apache.org/hadoop/Hive/PoweredBy?action=diff&rev1=16&rev2=17 -- * [[http://www.videoegg.com|VideoEgg]] We use Hive as the core database for our data warehouse where we track and analyze all the usage data of the ads across our network. + * TaoBao (www dot taobao dot com) + We use Hive for data mining, internal log analysis and ad-hoc queries. We also do some extensively developing work on Hive. +
svn commit: r916568 - in /hadoop/common/tags: release-0.20.2-rc1/ release-0.20.2-rc2/ release-0.20.2-rc3/
Author: cdouglas Date: Fri Feb 26 05:12:08 2010 New Revision: 916568 URL: http://svn.apache.org/viewvc?rev=916568&view=rev Log: Remove old 0.20.2 release candidates Removed: hadoop/common/tags/release-0.20.2-rc1/ hadoop/common/tags/release-0.20.2-rc2/ hadoop/common/tags/release-0.20.2-rc3/
svn commit: r916569 - in /hadoop/common/tags: release-0.20.2-rc4/ release-0.20.2/
Author: cdouglas Date: Fri Feb 26 05:13:14 2010 New Revision: 916569 URL: http://svn.apache.org/viewvc?rev=916569&view=rev Log: Hadoop 0.20.2 release Added: hadoop/common/tags/release-0.20.2/ (props changed) - copied from r916568, hadoop/common/tags/release-0.20.2-rc4/ Removed: hadoop/common/tags/release-0.20.2-rc4/ Propchange: hadoop/common/tags/release-0.20.2/ -- --- svn:ignore (added) +++ svn:ignore Fri Feb 26 05:13:14 2010 @@ -0,0 +1,7 @@ +build +logs +.classpath +.project +.settings + +.externalToolBuilders Propchange: hadoop/common/tags/release-0.20.2/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Fri Feb 26 05:13:14 2010 @@ -0,0 +1,3 @@ +/hadoop/common/trunk:910709 +/hadoop/core/branches/branch-0.19:713112 +/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746338,746902-746903,746925,746944,746968,746970,747279,747289,747802,748084,748090,748783,749262,749318,749863,750533,752073,752609,752834,752836,752913,752932,753112-753113,753346,754645,754847,754927,755035,755226,755348,755370,755418,755426,755790,755905,755938,755960,755986,755998,756352,757448,757624,757849,758156,758180,759398,759932,760502,760783,761046,761482,761632,762216,762879,763107,763502,764967,765016,765809,765951,771607,771661,772844,772876,772884,772920,773889,776638,778962,778966,779893,781720,784661,785046,785569
[Hadoop Wiki] Update of "Hive/Roadmap" by NamitJain
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "Hive/Roadmap" page has been changed by NamitJain. http://wiki.apache.org/hadoop/Hive/Roadmap?action=diff&rev1=21&rev2=22 -- * [[http://issues.apache.org/jira/browse/HIVE-870|semijoin]] * [[http://issues.apache.org/jira/browse/HIVE-655|UDTF]] * [[http://issues.apache.org/jira/browse/HIVE-31|Support for Create Table as Select (is available on trunk and versions later than 0.4.0)]] + * [[http://issues.apache.org/jira/browse/HIVE-931|Using sort and bucketing properties to optimize queries]] + * UNIQUE JOINS - that support a different semantics than the outer joins + * TypedBytes for user scripts + * [[Hive/ViewDev|Views]] for changing table names/columns without breaking existing queries [big] + * [[http://issues.apache.org/jira/browse/HIVE-917|Bucketed Map Join]] == Features working on now == * Hive CLI improvement/Error messages: @@ -23, +28 @@ * Bucketed Medium/Percentile * GraphViz for graphing operator tree * Multiple-partition inserts [big] - * [[Hive/ViewDev|Views]] for changing table names/columns without breaking existing queries [big] * GenericUDTF * Performance - * TypedBytes for user scripts + * [[http://issues.apache.org/jira/browse/HIVE-1194|Sort Merge Join]] * Hive Freeway * Allow Hive partition locations to be file/files. @@ -37, +41 @@ == More long-term Features (yet to be prioritized) == * Support for Indexes - * UNIQUE JOINS - that support a different semantics than the outer joins * Support for Insert Appends - * Using sort and bucketing properties to optimize queries * Support for IN, exists and correlated subqueries * More native types - Enums, timestamp * Passing schema to scripts through an environment variable
[Hadoop Wiki] Update of "Hive/Roadmap" by NamitJain
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "Hive/Roadmap" page has been changed by NamitJain. http://wiki.apache.org/hadoop/Hive/Roadmap?action=diff&rev1=22&rev2=23 -- * [[http://issues.apache.org/jira/browse/HIVE-31|Support for Create Table as Select (is available on trunk and versions later than 0.4.0)]] * [[http://issues.apache.org/jira/browse/HIVE-931|Using sort and bucketing properties to optimize queries]] * UNIQUE JOINS - that support a different semantics than the outer joins - * TypedBytes for user scripts + * [[http://issues.apache.org/jira/browse/HIVE-1023|TypedBytes for user scripts]] * [[Hive/ViewDev|Views]] for changing table names/columns without breaking existing queries [big] * [[http://issues.apache.org/jira/browse/HIVE-917|Bucketed Map Join]]
[Hadoop Wiki] Update of "Hive/Roadmap" by NamitJain
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "Hive/Roadmap" page has been changed by NamitJain. http://wiki.apache.org/hadoop/Hive/Roadmap?action=diff&rev1=23&rev2=24 -- * ODBC driver [[Hive/HiveODBC]] * [[http://issues.apache.org/jira/browse/HIVE-870|semijoin]] * [[http://issues.apache.org/jira/browse/HIVE-655|UDTF]] - * [[http://issues.apache.org/jira/browse/HIVE-31|Support for Create Table as Select (is available on trunk and versions later than 0.4.0)]] + * [[http://issues.apache.org/jira/browse/HIVE-31|Create Table as Select]] * [[http://issues.apache.org/jira/browse/HIVE-931|Using sort and bucketing properties to optimize queries]] - * UNIQUE JOINS - that support a different semantics than the outer joins + * [[http://issues.apache.org/jira/browse/HIVE-591|UNIQUE JOINS - that support a different semantics than the outer joins]] * [[http://issues.apache.org/jira/browse/HIVE-1023|TypedBytes for user scripts]] * [[Hive/ViewDev|Views]] for changing table names/columns without breaking existing queries [big] * [[http://issues.apache.org/jira/browse/HIVE-917|Bucketed Map Join]] + == Features working on now == * Hive CLI improvement/Error messages:
[Hadoop Wiki] Update of "Hive/Roadmap" by NamitJain
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "Hive/Roadmap" page has been changed by NamitJain. http://wiki.apache.org/hadoop/Hive/Roadmap?action=diff&rev1=24&rev2=25 -- * [[http://issues.apache.org/jira/browse/HIVE-1023|TypedBytes for user scripts]] * [[Hive/ViewDev|Views]] for changing table names/columns without breaking existing queries [big] * [[http://issues.apache.org/jira/browse/HIVE-917|Bucketed Map Join]] + * [[http://issues.apache.org/jira/browse/HIVE-74|Combine File Input Format]] == Features working on now ==