svn commit: r1549710 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java

2013-12-09 Thread jlowe
Author: jlowe
Date: Mon Dec  9 23:42:36 2013
New Revision: 1549710

URL: http://svn.apache.org/r1549710
Log:
svn merge -c 1549705 FIXES: MAPREDUCE-5656. bzip2 codec can drop records when 
reading data in splits. Contributed by Jason Lowe

Modified:

hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java

Modified: 
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java?rev=1549710&r1=1549709&r2=1549710&view=diff
==
--- 
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
 (original)
+++ 
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
 Mon Dec  9 23:42:36 2013
@@ -175,6 +175,11 @@ public class LineReader implements Close
 }
   }
 
+  protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter)
+  throws IOException {
+return in.read(buffer);
+  }
+
   /**
* Read a line terminated by one of CR, LF, or CRLF.
*/
@@ -208,7 +213,7 @@ public class LineReader implements Close
 if (prevCharCR) {
   ++bytesConsumed; //account for CR from previous read
 }
-bufferLength = in.read(buffer);
+bufferLength = fillBuffer(in, buffer, prevCharCR);
 if (bufferLength <= 0) {
   break; // EOF
 }
@@ -296,7 +301,7 @@ public class LineReader implements Close
   int startPosn = bufferPosn; // Start from previous end position
   if (bufferPosn >= bufferLength) {
 startPosn = bufferPosn = 0;
-bufferLength = in.read(buffer);
+bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0);
 if (bufferLength <= 0) {
   str.append(recordDelimiterBytes, 0, ambiguousByteCount);
   break; // EOF




svn commit: r1549705 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java

2013-12-09 Thread jlowe
Author: jlowe
Date: Mon Dec  9 23:31:30 2013
New Revision: 1549705

URL: http://svn.apache.org/r1549705
Log:
MAPREDUCE-5656. bzip2 codec can drop records when reading data in splits. 
Contributed by Jason Lowe

Modified:

hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java

Modified: 
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java?rev=1549705&r1=1549704&r2=1549705&view=diff
==
--- 
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
 (original)
+++ 
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java
 Mon Dec  9 23:31:30 2013
@@ -175,6 +175,11 @@ public class LineReader implements Close
 }
   }
 
+  protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter)
+  throws IOException {
+return in.read(buffer);
+  }
+
   /**
* Read a line terminated by one of CR, LF, or CRLF.
*/
@@ -208,7 +213,7 @@ public class LineReader implements Close
 if (prevCharCR) {
   ++bytesConsumed; //account for CR from previous read
 }
-bufferLength = in.read(buffer);
+bufferLength = fillBuffer(in, buffer, prevCharCR);
 if (bufferLength <= 0) {
   break; // EOF
 }
@@ -296,7 +301,7 @@ public class LineReader implements Close
   int startPosn = bufferPosn; // Start from previous end position
   if (bufferPosn >= bufferLength) {
 startPosn = bufferPosn = 0;
-bufferLength = in.read(buffer);
+bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0);
 if (bufferLength <= 0) {
   str.append(recordDelimiterBytes, 0, ambiguousByteCount);
   break; // EOF




svn commit: r1549699 - in /hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common: ./ src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/se

2013-12-09 Thread cnauroth
Author: cnauroth
Date: Mon Dec  9 22:52:02 2013
New Revision: 1549699

URL: http://svn.apache.org/r1549699
Log:
Merge trunk to HDFS-4685.

Modified:

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
   (contents, props changed)

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
   (props changed)

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
   (props changed)

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/
   (props changed)

hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java

Modified: 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549699&r1=1549698&r2=1549699&view=diff
==
--- 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
 (original)
+++ 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
 Mon Dec  9 22:52:02 2013
@@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED
 
   OPTIMIZATIONS
 
+HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who"
+(vinay via cmccabe)
+
   BUG FIXES
 
 HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9)

Propchange: 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt
--
  Merged 
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1548386-1549698

Propchange: 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/
--
  Merged 
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1548386-1549698

Propchange: 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/
--
  Merged 
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1548386-1549698

Modified: 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==
--- 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
 (original)
+++ 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
 Mon Dec  9 22:52:02 2013
@@ -204,6 +204,14 @@ public class CommonConfigurationKeys ext
   public static final String DEFAULT_HADOOP_HTTP_STATIC_USER =
 "dr.who";
 
+  /**
+   * User->groups static mapping to override the groups lookup
+   */
+  public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES = 
+  "hadoop.user.group.static.mapping.overrides";
+  public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT =
+  "dr.who=;";
+
   /** Enable/Disable aliases serving from jetty */
   public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES =
 "hadoop.jetty.logs.serve.aliases";

Modified: 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==
--- 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 (original)
+++ 
hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 Mon Dec  9 22:52:02 2013
@@ -18,15 +18,20 @@
 package org.apache.hadoop.security;
 
 import java.io.IOException;
+impo

svn commit: r1549699 - in /hadoop/common/branches/HDFS-4685: ./ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java

2013-12-09 Thread cnauroth
Author: cnauroth
Date: Mon Dec  9 22:52:02 2013
New Revision: 1549699

URL: http://svn.apache.org/r1549699
Log:
Merge trunk to HDFS-4685.

Modified:
hadoop/common/branches/HDFS-4685/   (props changed)

hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java

Propchange: hadoop/common/branches/HDFS-4685/
--
  Merged /hadoop/common/trunk:r1548386-1549698

Modified: 
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1549699&r1=1549698&r2=1549699&view=diff
==
--- 
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
 (original)
+++ 
hadoop/common/branches/HDFS-4685/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
 Mon Dec  9 22:52:02 2013
@@ -867,5 +867,10 @@ public class ResourceSchedulerWrapper im
   ApplicationAttemptId appAttemptId) {
 return scheduler.getAppResourceUsageReport(appAttemptId);
   }
+
+  @Override
+  public List getAppsInQueue(String queue) {
+return scheduler.getAppsInQueue(queue);
+  }
 }
 




svn commit: r1549648 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java

2013-12-09 Thread jeagles
Author: jeagles
Date: Mon Dec  9 19:12:52 2013
New Revision: 1549648

URL: http://svn.apache.org/r1549648
Log:
HADOOP-10148. backport hadoop-10107 to branch-0.23 (Chen He via jeagles)

Modified:

hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt

hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

Modified: 
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549648&r1=1549647&r2=1549648&view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
 Mon Dec  9 19:12:52 2013
@@ -8,6 +8,8 @@ Release 0.23.11 - UNRELEASED
 
   IMPROVEMENTS
 
+HADOOP-10148. backport hadoop-10107 to branch-0.23 (Chen He via jeagles)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: 
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1549648&r1=1549647&r2=1549648&view=diff
==
--- 
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 (original)
+++ 
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 Mon Dec  9 19:12:52 2013
@@ -1600,6 +1600,7 @@ public abstract class Server {
 // Start the listener here and let it bind to the port
 listener = new Listener();
 this.port = listener.getAddress().getPort();
+connectionManager = new ConnectionManager();
 this.rpcMetrics = RpcMetrics.create(this);
 this.rpcDetailedMetrics = RpcDetailedMetrics.create(this.port);
 this.tcpNoDelay = conf.getBoolean(
@@ -1608,7 +1609,6 @@ public abstract class Server {
 
 // Create the responder here
 responder = new Responder();
-connectionManager = new ConnectionManager();
 
 if (isSecurityEnabled) {
   SaslRpcServer.init(conf);




svn commit: r1549626 - in /hadoop/common/branches/HDFS-2832: ./ hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java

2013-12-09 Thread arp
Author: arp
Date: Mon Dec  9 17:38:20 2013
New Revision: 1549626

URL: http://svn.apache.org/r1549626
Log:
Merging r1548329 through r1549625 from trunk to branch HDFS-2832

Modified:
hadoop/common/branches/HDFS-2832/   (props changed)

hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java

Propchange: hadoop/common/branches/HDFS-2832/
--
  Merged /hadoop/common/trunk:r1548329-1549625

Modified: 
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1549626&r1=1549625&r2=1549626&view=diff
==
--- 
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
 (original)
+++ 
hadoop/common/branches/HDFS-2832/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
 Mon Dec  9 17:38:20 2013
@@ -867,5 +867,10 @@ public class ResourceSchedulerWrapper im
   ApplicationAttemptId appAttemptId) {
 return scheduler.getAppResourceUsageReport(appAttemptId);
   }
+
+  @Override
+  public List getAppsInQueue(String queue) {
+return scheduler.getAppsInQueue(queue);
+  }
 }
 




svn commit: r1549626 - in /hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common: ./ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/security/ src/mai

2013-12-09 Thread arp
Author: arp
Date: Mon Dec  9 17:38:20 2013
New Revision: 1549626

URL: http://svn.apache.org/r1549626
Log:
Merging r1548329 through r1549625 from trunk to branch HDFS-2832

Modified:

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
   (contents, props changed)

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
   (props changed)

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java

Modified: 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549626&r1=1549625&r2=1549626&view=diff
==
--- 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
 (original)
+++ 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
 Mon Dec  9 17:38:20 2013
@@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED
 
   OPTIMIZATIONS
 
+HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who"
+(vinay via cmccabe)
+
   BUG FIXES
 
 HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9)

Propchange: 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/CHANGES.txt
--
  Merged 
/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1548329-1549625

Propchange: 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/
--
  Merged 
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1548329-1549625

Modified: 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1549626&r1=1549625&r2=1549626&view=diff
==
--- 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
 (original)
+++ 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
 Mon Dec  9 17:38:20 2013
@@ -204,6 +204,14 @@ public class CommonConfigurationKeys ext
   public static final String DEFAULT_HADOOP_HTTP_STATIC_USER =
 "dr.who";
 
+  /**
+   * User->groups static mapping to override the groups lookup
+   */
+  public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES = 
+  "hadoop.user.group.static.mapping.overrides";
+  public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT =
+  "dr.who=;";
+
   /** Enable/Disable aliases serving from jetty */
   public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES =
 "hadoop.jetty.logs.serve.aliases";

Modified: 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
URL: 
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1549626&r1=1549625&r2=1549626&view=diff
==
--- 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 (original)
+++ 
hadoop/common/branches/HDFS-2832/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
 Mon Dec  9 17:38:20 2013
@@ -18,15 +18,20 @@
 package org.apache.hadoop.security;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys