svn commit: r1549710 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
Author: jlowe Date: Mon Dec 9 23:42:36 2013 New Revision: 1549710 URL: http://svn.apache.org/r1549710 Log: svn merge -c 1549705 FIXES: MAPREDUCE-5656. bzip2 codec can drop records when reading data in splits. Contributed by Jason Lowe Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java?rev=1549710&r1=1549709&r2=1549710&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Mon Dec 9 23:42:36 2013 @@ -175,6 +175,11 @@ public class LineReader implements Close } } + protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) + throws IOException { +return in.read(buffer); + } + /** * Read a line terminated by one of CR, LF, or CRLF. */ @@ -208,7 +213,7 @@ public class LineReader implements Close if (prevCharCR) { ++bytesConsumed; //account for CR from previous read } -bufferLength = in.read(buffer); +bufferLength = fillBuffer(in, buffer, prevCharCR); if (bufferLength <= 0) { break; // EOF } @@ -296,7 +301,7 @@ public class LineReader implements Close int startPosn = bufferPosn; // Start from previous end position if (bufferPosn >= bufferLength) { startPosn = bufferPosn = 0; -bufferLength = in.read(buffer); +bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0); if (bufferLength <= 0) { str.append(recordDelimiterBytes, 0, ambiguousByteCount); break; // EOF
svn commit: r1549705 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
Author: jlowe Date: Mon Dec 9 23:31:30 2013 New Revision: 1549705 URL: http://svn.apache.org/r1549705 Log: MAPREDUCE-5656. bzip2 codec can drop records when reading data in splits. Contributed by Jason Lowe Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java?rev=1549705&r1=1549704&r2=1549705&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Mon Dec 9 23:31:30 2013 @@ -175,6 +175,11 @@ public class LineReader implements Close } } + protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) + throws IOException { +return in.read(buffer); + } + /** * Read a line terminated by one of CR, LF, or CRLF. */ @@ -208,7 +213,7 @@ public class LineReader implements Close if (prevCharCR) { ++bytesConsumed; //account for CR from previous read } -bufferLength = in.read(buffer); +bufferLength = fillBuffer(in, buffer, prevCharCR); if (bufferLength <= 0) { break; // EOF } @@ -296,7 +301,7 @@ public class LineReader implements Close int startPosn = bufferPosn; // Start from previous end position if (bufferPosn >= bufferLength) { startPosn = bufferPosn = 0; -bufferLength = in.read(buffer); +bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0); if (bufferLength <= 0) { str.append(recordDelimiterBytes, 0, ambiguousByteCount); break; // EOF
svn commit: r1549699 - in /hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common: ./ src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/se
Author: cnauroth Date: Mon Dec 9 22:52:02 2013 New Revision: 1549699 URL: http://svn.apache.org/r1549699 Log: Merge trunk to HDFS-4685. Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549699&r1=1549698&r2=1549699&view=diff == --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt Mon Dec 9 22:52:02 2013 @@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED OPTIMIZATIONS +HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who" +(vinay via cmccabe) + BUG FIXES HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9) Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt -- Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1548386-1549698 Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ -- Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1548386-1549698 Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ -- Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1548386-1549698 Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1549699&r1=1549698&r2=1549699&view=diff == --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon Dec 9 22:52:02 2013 @@ -204,6 +204,14 @@ public class CommonConfigurationKeys ext public static final String DEFAULT_HADOOP_HTTP_STATIC_USER = "dr.who"; + /** + * User->groups static mapping to override the groups lookup + */ + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES = + "hadoop.user.group.static.mapping.overrides"; + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT = + "dr.who=;"; + /** Enable/Disable aliases serving from jetty */ public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES = "hadoop.jetty.logs.serve.aliases"; Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1549699&r1=1549698&r2=1549699&view=diff == --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java Mon Dec 9 22:52:02 2013 @@ -18,15 +18,20 @@ package org.apache.hadoop.security; import java.io.IOException; +impo
svn commit: r1549699 - in /hadoop/common/branches/HDFS-4685: ./ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Author: cnauroth Date: Mon Dec 9 22:52:02 2013 New Revision: 1549699 URL: http://svn.apache.org/r1549699 Log: Merge trunk to HDFS-4685. Modified: hadoop/common/branches/HDFS-4685/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Propchange: hadoop/common/branches/HDFS-4685/ -- Merged /hadoop/common/trunk:r1548386-1549698 Modified: hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1549699&r1=1549698&r2=1549699&view=diff == --- hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Mon Dec 9 22:52:02 2013 @@ -867,5 +867,10 @@ public class ResourceSchedulerWrapper im ApplicationAttemptId appAttemptId) { return scheduler.getAppResourceUsageReport(appAttemptId); } + + @Override + public List getAppsInQueue(String queue) { +return scheduler.getAppsInQueue(queue); + } }
svn commit: r1549648 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java
Author: jeagles Date: Mon Dec 9 19:12:52 2013 New Revision: 1549648 URL: http://svn.apache.org/r1549648 Log: HADOOP-10148. backport hadoop-10107 to branch-0.23 (Chen He via jeagles) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549648&r1=1549647&r2=1549648&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Dec 9 19:12:52 2013 @@ -8,6 +8,8 @@ Release 0.23.11 - UNRELEASED IMPROVEMENTS +HADOOP-10148. backport hadoop-10107 to branch-0.23 (Chen He via jeagles) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1549648&r1=1549647&r2=1549648&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Mon Dec 9 19:12:52 2013 @@ -1600,6 +1600,7 @@ public abstract class Server { // Start the listener here and let it bind to the port listener = new Listener(); this.port = listener.getAddress().getPort(); +connectionManager = new ConnectionManager(); this.rpcMetrics = RpcMetrics.create(this); this.rpcDetailedMetrics = RpcDetailedMetrics.create(this.port); this.tcpNoDelay = conf.getBoolean( @@ -1608,7 +1609,6 @@ public abstract class Server { // Create the responder here responder = new Responder(); -connectionManager = new ConnectionManager(); if (isSecurityEnabled) { SaslRpcServer.init(conf);
svn commit: r1549626 - in /hadoop/common/branches/HDFS-2832: ./ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Author: arp Date: Mon Dec 9 17:38:20 2013 New Revision: 1549626 URL: http://svn.apache.org/r1549626 Log: Merging r1548329 through r1549625 from trunk to branch HDFS-2832 Modified: hadoop/common/branches/HDFS-2832/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Propchange: hadoop/common/branches/HDFS-2832/ -- Merged /hadoop/common/trunk:r1548329-1549625 Modified: hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1549626&r1=1549625&r2=1549626&view=diff == --- hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Mon Dec 9 17:38:20 2013 @@ -867,5 +867,10 @@ public class ResourceSchedulerWrapper im ApplicationAttemptId appAttemptId) { return scheduler.getAppResourceUsageReport(appAttemptId); } + + @Override + public List getAppsInQueue(String queue) { +return scheduler.getAppsInQueue(queue); + } }
svn commit: r1549626 - in /hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common: ./ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/security/ src/mai
Author: arp Date: Mon Dec 9 17:38:20 2013 New Revision: 1549626 URL: http://svn.apache.org/r1549626 Log: Merging r1548329 through r1549625 from trunk to branch HDFS-2832 Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549626&r1=1549625&r2=1549626&view=diff == --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt Mon Dec 9 17:38:20 2013 @@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED OPTIMIZATIONS +HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who" +(vinay via cmccabe) + BUG FIXES HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9) Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt -- Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1548329-1549625 Propchange: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/ -- Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1548329-1549625 Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1549626&r1=1549625&r2=1549626&view=diff == --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon Dec 9 17:38:20 2013 @@ -204,6 +204,14 @@ public class CommonConfigurationKeys ext public static final String DEFAULT_HADOOP_HTTP_STATIC_USER = "dr.who"; + /** + * User->groups static mapping to override the groups lookup + */ + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES = + "hadoop.user.group.static.mapping.overrides"; + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT = + "dr.who=;"; + /** Enable/Disable aliases serving from jetty */ public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES = "hadoop.jetty.logs.serve.aliases"; Modified: hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1549626&r1=1549625&r2=1549626&view=diff == --- hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java Mon Dec 9 17:38:20 2013 @@ -18,15 +18,20 @@ package org.apache.hadoop.security; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys