hadoop git commit: YARN-644: Basic null check is not performed on passed in arguments before using them in ContainerManagerImpl.startContainer
Repository: hadoop
Updated Branches:
refs/heads/trunk dc2b2ae31 - bcf289050
YARN-644: Basic null check is not performed on passed in arguments before using
them in ContainerManagerImpl.startContainer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcf28905
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcf28905
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcf28905
Branch: refs/heads/trunk
Commit: bcf2890502fbd11dd394048fe30d67c92aeec4fa
Parents: dc2b2ae
Author: Robert (Bobby) Evans ev...@yahoo-inc.com
Authored: Fri May 8 11:09:29 2015 -0500
Committer: Robert (Bobby) Evans ev...@yahoo-inc.com
Committed: Fri May 8 11:11:01 2015 -0500
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../containermanager/ContainerManagerImpl.java | 27 +-
.../BaseContainerManagerTest.java | 6 ++
.../containermanager/TestContainerManager.java | 87
4 files changed, 122 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcf28905/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 26febe9..7f6a09f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -113,6 +113,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+YARN-644. Basic null check is not performed on passed in arguments before
+using them in ContainerManagerImpl.startContainer (Varun Saxena via bobby)
+
YARN-1880. Cleanup TestApplicationClientProtocolOnHA
(ozawa via harsh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcf28905/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index acac600..c48df64 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -151,6 +151,10 @@ public class ContainerManagerImpl extends CompositeService
implements
private static final Log LOG = LogFactory.getLog(ContainerManagerImpl.class);
+ static final String INVALID_NMTOKEN_MSG = Invalid NMToken;
+ static final String INVALID_CONTAINERTOKEN_MSG =
+ Invalid ContainerToken;
+
final Context context;
private final ContainersMonitor containersMonitor;
private Server server;
@@ -641,6 +645,9 @@ public class ContainerManagerImpl extends CompositeService
implements
protected void authorizeUser(UserGroupInformation remoteUgi,
NMTokenIdentifier nmTokenIdentifier) throws YarnException {
+if (nmTokenIdentifier == null) {
+ throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG);
+}
if (!remoteUgi.getUserName().equals(
nmTokenIdentifier.getApplicationAttemptId().toString())) {
throw RPCUtil.getRemoteException(Expected applicationAttemptId:
@@ -658,7 +665,12 @@ public class ContainerManagerImpl extends CompositeService
implements
@VisibleForTesting
protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
-
+if (nmTokenIdentifier == null) {
+ throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG);
+}
+if (containerTokenIdentifier == null) {
+ throw RPCUtil.getRemoteException(INVALID_CONTAINERTOKEN_MSG);
+}
ContainerId containerId = containerTokenIdentifier.getContainerID();
String containerIDStr = containerId.toString();
boolean unauthorized = false;
@@ -717,6 +729,10 @@ public class ContainerManagerImpl extends CompositeService
implements
for (StartContainerRequest request : requests.getStartContainerRequests())
{
ContainerId containerId = null;
try {
+if (request.getContainerToken() == null ||
+request.getContainerToken().getIdentifier() == null) {
+ throw new IOException(INVALID_CONTAINERTOKEN_MSG
hadoop git commit: YARN-644: Basic null check is not performed on passed in arguments before using them in ContainerManagerImpl.startContainer
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a957baeb7 - 28e0593b9
YARN-644: Basic null check is not performed on passed in arguments before using
them in ContainerManagerImpl.startContainer
(cherry picked from commit bcf2890502fbd11dd394048fe30d67c92aeec4fa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28e0593b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28e0593b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28e0593b
Branch: refs/heads/branch-2
Commit: 28e0593b966dd1e288828be8639200be2aa7658e
Parents: a957bae
Author: Robert (Bobby) Evans ev...@yahoo-inc.com
Authored: Fri May 8 11:09:29 2015 -0500
Committer: Robert (Bobby) Evans ev...@yahoo-inc.com
Committed: Fri May 8 11:11:53 2015 -0500
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../containermanager/ContainerManagerImpl.java | 27 +-
.../BaseContainerManagerTest.java | 6 ++
.../containermanager/TestContainerManager.java | 87
4 files changed, 122 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28e0593b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 32c5ae1..5ae87ef 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -65,6 +65,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+YARN-644. Basic null check is not performed on passed in arguments before
+using them in ContainerManagerImpl.startContainer (Varun Saxena via bobby)
+
YARN-1880. Cleanup TestApplicationClientProtocolOnHA
(ozawa via harsh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28e0593b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index acac600..c48df64 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -151,6 +151,10 @@ public class ContainerManagerImpl extends CompositeService
implements
private static final Log LOG = LogFactory.getLog(ContainerManagerImpl.class);
+ static final String INVALID_NMTOKEN_MSG = Invalid NMToken;
+ static final String INVALID_CONTAINERTOKEN_MSG =
+ Invalid ContainerToken;
+
final Context context;
private final ContainersMonitor containersMonitor;
private Server server;
@@ -641,6 +645,9 @@ public class ContainerManagerImpl extends CompositeService
implements
protected void authorizeUser(UserGroupInformation remoteUgi,
NMTokenIdentifier nmTokenIdentifier) throws YarnException {
+if (nmTokenIdentifier == null) {
+ throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG);
+}
if (!remoteUgi.getUserName().equals(
nmTokenIdentifier.getApplicationAttemptId().toString())) {
throw RPCUtil.getRemoteException(Expected applicationAttemptId:
@@ -658,7 +665,12 @@ public class ContainerManagerImpl extends CompositeService
implements
@VisibleForTesting
protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
ContainerTokenIdentifier containerTokenIdentifier) throws YarnException {
-
+if (nmTokenIdentifier == null) {
+ throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG);
+}
+if (containerTokenIdentifier == null) {
+ throw RPCUtil.getRemoteException(INVALID_CONTAINERTOKEN_MSG);
+}
ContainerId containerId = containerTokenIdentifier.getContainerID();
String containerIDStr = containerId.toString();
boolean unauthorized = false;
@@ -717,6 +729,10 @@ public class ContainerManagerImpl extends CompositeService
implements
for (StartContainerRequest request : requests.getStartContainerRequests())
{
ContainerId containerId = null;
try {
+if (request.getContainerToken() == null ||
+request.getContainerToken().getIdentifier() == null) {
+ throw new
hadoop git commit: HADOOP-6842. hadoop fs -text does not give a useful text representation of MapWritable objects
Repository: hadoop
Updated Branches:
refs/heads/trunk 5d708a472 - cc17c8358
HADOOP-6842. hadoop fs -text does not give a useful text representation of
MapWritable objects
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc17c835
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc17c835
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc17c835
Branch: refs/heads/trunk
Commit: cc17c83585f581d56e7c42b91464cd7a99e8cf90
Parents: 5d708a4
Author: Robert (Bobby) Evans ev...@yahoo-inc.com
Authored: Fri May 8 11:56:59 2015 -0500
Committer: Robert (Bobby) Evans ev...@yahoo-inc.com
Committed: Fri May 8 11:57:14 2015 -0500
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/io/MapWritable.java | 5 +
.../test/java/org/apache/hadoop/io/TestMapWritable.java | 11 ++-
3 files changed, 18 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc17c835/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f4b9246..432c6b6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -479,6 +479,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+HADOOP-6842. hadoop fs -text does not give a useful text representation
+of MapWritable objects (Akira Ajisaka via bobby)
+
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
GenericOptionsParser default help text.
(Brahma Reddy Battula via harsh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc17c835/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
index fec168b..0379ec2 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
@@ -189,4 +189,9 @@ public class MapWritable extends AbstractMapWritable
instance.put(key, value);
}
}
+
+ @Override
+ public String toString() {
+return instance.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc17c835/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
index 9284949..4597b90 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
@@ -148,5 +148,14 @@ public class TestMapWritable extends TestCase {
assertEquals(map1.hashCode(), map2.hashCode());
assertFalse(map1.hashCode() == map3.hashCode());
-}
+ }
+
+ /** Verify text command outputs a useful representation for MapWritable. */
+ public void testToString() {
+MapWritable map = new MapWritable();
+final IntWritable key = new IntWritable(5);
+final Text value = new Text(value);
+map.put(key, value);
+assertEquals({5=value}, map.toString());
+ }
}
hadoop git commit: HADOOP-6842. hadoop fs -text does not give a useful text representation of MapWritable objects
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d4d7d7bf2 - eef8d70bb
HADOOP-6842. hadoop fs -text does not give a useful text representation of
MapWritable objects
(cherry picked from commit cc17c83585f581d56e7c42b91464cd7a99e8cf90)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eef8d70b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eef8d70b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eef8d70b
Branch: refs/heads/branch-2
Commit: eef8d70bb0121efe8d8aafd2b80fb4c75ddad079
Parents: d4d7d7b
Author: Robert (Bobby) Evans ev...@yahoo-inc.com
Authored: Fri May 8 11:56:59 2015 -0500
Committer: Robert (Bobby) Evans ev...@yahoo-inc.com
Committed: Fri May 8 12:00:31 2015 -0500
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/io/MapWritable.java | 5 +
.../test/java/org/apache/hadoop/io/TestMapWritable.java | 11 ++-
3 files changed, 18 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eef8d70b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2c358bb..8623a3e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -19,6 +19,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+HADOOP-6842. hadoop fs -text does not give a useful text representation
+of MapWritable objects (Akira Ajisaka via bobby)
+
HADOOP-11719. [Fsshell] Remove bin/hadoop reference from
GenericOptionsParser default help text.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eef8d70b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
index fec168b..0379ec2 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapWritable.java
@@ -189,4 +189,9 @@ public class MapWritable extends AbstractMapWritable
instance.put(key, value);
}
}
+
+ @Override
+ public String toString() {
+return instance.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eef8d70b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
index 9284949..4597b90 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestMapWritable.java
@@ -148,5 +148,14 @@ public class TestMapWritable extends TestCase {
assertEquals(map1.hashCode(), map2.hashCode());
assertFalse(map1.hashCode() == map3.hashCode());
-}
+ }
+
+ /** Verify text command outputs a useful representation for MapWritable. */
+ public void testToString() {
+MapWritable map = new MapWritable();
+final IntWritable key = new IntWritable(5);
+final Text value = new Text(value);
+map.put(key, value);
+assertEquals({5=value}, map.toString());
+ }
}
svn commit: r1552104 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Author: bobby
Date: Wed Dec 18 21:20:56 2013
New Revision: 1552104
URL: http://svn.apache.org/r1552104
Log:
HADOOP-10164. Allow UGI to login with a known Subject (bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1552104r1=1552103r2=1552104view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Dec
18 21:20:56 2013
@@ -399,6 +399,8 @@ Release 2.4.0 - UNRELEASED
HADOOP-10168. fix javadoc of ReflectionUtils#copy. (Thejas Nair via suresh)
+HADOOP-10164. Allow UGI to login with a known Subject (bobby)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1552104r1=1552103r2=1552104view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Wed Dec 18 21:20:56 2013
@@ -477,7 +477,7 @@ public class UserGroupInformation {
private static final AppConfigurationEntry[] SIMPLE_CONF =
new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, HADOOP_LOGIN};
-
+
private static final AppConfigurationEntry[] USER_KERBEROS_CONF =
new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, USER_KERBEROS_LOGIN,
HADOOP_LOGIN};
@@ -682,44 +682,59 @@ public class UserGroupInformation {
public synchronized
static UserGroupInformation getLoginUser() throws IOException {
if (loginUser == null) {
- ensureInitialized();
- try {
-Subject subject = new Subject();
-LoginContext login =
-newLoginContext(authenticationMethod.getLoginAppName(),
-subject, new HadoopConfiguration());
-login.login();
-UserGroupInformation realUser = new UserGroupInformation(subject);
-realUser.setLogin(login);
-realUser.setAuthenticationMethod(authenticationMethod);
-realUser = new UserGroupInformation(login.getSubject());
-// If the HADOOP_PROXY_USER environment variable or property
-// is specified, create a proxy user as the logged in user.
-String proxyUser = System.getenv(HADOOP_PROXY_USER);
-if (proxyUser == null) {
- proxyUser = System.getProperty(HADOOP_PROXY_USER);
-}
-loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser,
realUser);
-
-String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
-if (fileLocation != null) {
- // Load the token storage file and put all of the tokens into the
- // user. Don't use the FileSystem API for reading since it has a lock
- // cycle (HADOOP-9212).
- Credentials cred = Credentials.readTokenStorageFile(
- new File(fileLocation), conf);
- loginUser.addCredentials(cred);
-}
-loginUser.spawnAutoRenewalThreadForUserCreds();
- } catch (LoginException le) {
-LOG.debug(failure to login, le);
-throw new IOException(failure to login, le);
+ loginUserFromSubject(null);
+}
+return loginUser;
+ }
+
+ /**
+ * Log in a user using the given subject
+ * @parma subject the subject to use when logging in a user, or null to
+ * create a new subject.
+ * @throws IOException if login fails
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public synchronized
+ static void loginUserFromSubject(Subject subject) throws IOException {
+ensureInitialized();
+try {
+ if (subject == null) {
+subject = new Subject();
+ }
+ LoginContext login =
+ newLoginContext(authenticationMethod.getLoginAppName(),
+ subject, new HadoopConfiguration());
+ login.login();
+ UserGroupInformation realUser = new UserGroupInformation(subject);
+ realUser.setLogin(login);
+ realUser.setAuthenticationMethod(authenticationMethod);
+ realUser = new UserGroupInformation(login.getSubject
svn commit: r1552106 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Author: bobby
Date: Wed Dec 18 21:26:13 2013
New Revision: 1552106
URL: http://svn.apache.org/r1552106
Log:
svn merge -c 1552104 FIXES: HADOOP-10164. Allow UGI to login with a known
Subject (bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1552106r1=1552105r2=1552106view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Dec 18 21:26:13 2013
@@ -107,6 +107,8 @@ Release 2.4.0 - UNRELEASED
HADOOP-10168. fix javadoc of ReflectionUtils#copy. (Thejas Nair via suresh)
+HADOOP-10164. Allow UGI to login with a known Subject (bobby)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1552106r1=1552105r2=1552106view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Wed Dec 18 21:26:13 2013
@@ -465,7 +465,7 @@ public class UserGroupInformation {
private static final AppConfigurationEntry[] SIMPLE_CONF =
new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, HADOOP_LOGIN};
-
+
private static final AppConfigurationEntry[] USER_KERBEROS_CONF =
new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, USER_KERBEROS_LOGIN,
HADOOP_LOGIN};
@@ -670,44 +670,59 @@ public class UserGroupInformation {
public synchronized
static UserGroupInformation getLoginUser() throws IOException {
if (loginUser == null) {
- ensureInitialized();
- try {
-Subject subject = new Subject();
-LoginContext login =
-newLoginContext(authenticationMethod.getLoginAppName(),
-subject, new HadoopConfiguration());
-login.login();
-UserGroupInformation realUser = new UserGroupInformation(subject);
-realUser.setLogin(login);
-realUser.setAuthenticationMethod(authenticationMethod);
-realUser = new UserGroupInformation(login.getSubject());
-// If the HADOOP_PROXY_USER environment variable or property
-// is specified, create a proxy user as the logged in user.
-String proxyUser = System.getenv(HADOOP_PROXY_USER);
-if (proxyUser == null) {
- proxyUser = System.getProperty(HADOOP_PROXY_USER);
-}
-loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser,
realUser);
-
-String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
-if (fileLocation != null) {
- // Load the token storage file and put all of the tokens into the
- // user. Don't use the FileSystem API for reading since it has a lock
- // cycle (HADOOP-9212).
- Credentials cred = Credentials.readTokenStorageFile(
- new File(fileLocation), conf);
- loginUser.addCredentials(cred);
-}
-loginUser.spawnAutoRenewalThreadForUserCreds();
- } catch (LoginException le) {
-LOG.debug(failure to login, le);
-throw new IOException(failure to login, le);
+ loginUserFromSubject(null);
+}
+return loginUser;
+ }
+
+ /**
+ * Log in a user using the given subject
+ * @parma subject the subject to use when logging in a user, or null to
+ * create a new subject.
+ * @throws IOException if login fails
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public synchronized
+ static void loginUserFromSubject(Subject subject) throws IOException {
+ensureInitialized();
+try {
+ if (subject == null) {
+subject = new Subject();
+ }
+ LoginContext login =
+ newLoginContext(authenticationMethod.getLoginAppName(),
+ subject, new HadoopConfiguration());
+ login.login();
+ UserGroupInformation realUser = new UserGroupInformation(subject);
+ realUser.setLogin(login
svn commit: r1552112 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Author: bobby
Date: Wed Dec 18 21:34:10 2013
New Revision: 1552112
URL: http://svn.apache.org/r1552112
Log:
HADOOP-10164. Allow UGI to login with a known Subject (bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1552112r1=1552111r2=1552112view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Dec 18 21:34:10 2013
@@ -10,6 +10,8 @@ Release 0.23.11 - UNRELEASED
HADOOP-10148. backport hadoop-10107 to branch-0.23 (Chen He via jeagles)
+HADOOP-10164. Allow UGI to login with a known Subject (bobby)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1552112r1=1552111r2=1552112view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Wed Dec 18 21:34:10 2013
@@ -435,7 +435,7 @@ public class UserGroupInformation {
private static final AppConfigurationEntry[] SIMPLE_CONF =
new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, HADOOP_LOGIN};
-
+
private static final AppConfigurationEntry[] USER_KERBEROS_CONF =
new AppConfigurationEntry[]{OS_SPECIFIC_LOGIN, USER_KERBEROS_LOGIN,
HADOOP_LOGIN};
@@ -525,49 +525,65 @@ public class UserGroupInformation {
public synchronized
static UserGroupInformation getLoginUser() throws IOException {
if (loginUser == null) {
- try {
-Subject subject = new Subject();
-LoginContext login;
-if (isSecurityEnabled()) {
- login =
newLoginContext(HadoopConfiguration.USER_KERBEROS_CONFIG_NAME,
- subject);
-} else {
- login = newLoginContext(HadoopConfiguration.SIMPLE_CONFIG_NAME,
- subject);
-}
-login.login();
-UserGroupInformation realUser = new UserGroupInformation(subject);
-realUser.setLogin(login);
-realUser.setAuthenticationMethod(isSecurityEnabled() ?
- AuthenticationMethod.KERBEROS :
- AuthenticationMethod.SIMPLE);
-realUser = new UserGroupInformation(login.getSubject());
-// If the HADOOP_PROXY_USER environment variable or property
-// is specified, create a proxy user as the logged in user.
-String proxyUser = System.getenv(HADOOP_PROXY_USER);
-if (proxyUser == null) {
- proxyUser = System.getProperty(HADOOP_PROXY_USER);
-}
-setLoginUser(proxyUser == null ? realUser : createProxyUser(proxyUser,
realUser));
-
-String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
-if (fileLocation != null) {
- // Load the token storage file and put all of the tokens into the
- // user. Don't use the FileSystem API for reading since it has a lock
- // cycle (HADOOP-9212).
- Credentials cred = Credentials.readTokenStorageFile(
- new File(fileLocation), conf);
- loginUser.addCredentials(cred);
-}
-loginUser.spawnAutoRenewalThreadForUserCreds();
- } catch (LoginException le) {
-throw new IOException(failure to login, le);
+ loginUserFromSubject(null);
+}
+return loginUser;
+ }
+
+ /**
+ * Log in a user using the given subject
+ * @parma subject the subject to use when logging in a user, or null to
+ * create a new subject.
+ * @throws IOException if login fails
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public synchronized
+ static void loginUserFromSubject(Subject subject) throws IOException {
+ensureInitialized();
+try {
+ if (subject == null) {
+subject = new Subject();
+ }
+ LoginContext login;
+ if (isSecurityEnabled()) {
+login = newLoginContext
svn commit: r1448745 - /hadoop/common/trunk/dev-support/test-patch.sh
Author: bobby
Date: Thu Feb 21 17:52:36 2013
New Revision: 1448745
URL: http://svn.apache.org/r1448745
Log:
amendment to HADOOP-9112 fix return codes (Surenkumar Nihalani via bobby)
Modified:
hadoop/common/trunk/dev-support/test-patch.sh
Modified: hadoop/common/trunk/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/dev-support/test-patch.sh?rev=1448745r1=1448744r2=1448745view=diff
==
--- hadoop/common/trunk/dev-support/test-patch.sh (original)
+++ hadoop/common/trunk/dev-support/test-patch.sh Thu Feb 21 17:52:36 2013
@@ -370,12 +370,12 @@ checkTests () {
JIRA_COMMENT=$JIRA_COMMENT
{color:green}+1 tests included appear to have a timeout.{color}
- return 1
+ return 0
fi
JIRA_COMMENT=$JIRA_COMMENT
{color:red}-1 one of tests included doesn't have a timeout.{color}
- return 0
+ return 1
}
cleanUpXml () {
svn commit: r1448285 - /hadoop/common/trunk/dev-support/test-patch.sh
Author: bobby
Date: Wed Feb 20 16:41:57 2013
New Revision: 1448285
URL: http://svn.apache.org/r1448285
Log:
HADOOP-9112. test-patch should -1 for @Tests without a timeout (Surenkumar
Nihalani via bobby)
Modified:
hadoop/common/trunk/dev-support/test-patch.sh
Modified: hadoop/common/trunk/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/dev-support/test-patch.sh?rev=1448285r1=1448284r2=1448285view=diff
==
--- hadoop/common/trunk/dev-support/test-patch.sh (original)
+++ hadoop/common/trunk/dev-support/test-patch.sh Wed Feb 20 16:41:57 2013
@@ -323,7 +323,7 @@ checkAuthor () {
}
###
-### Check for tests in the patch
+### Check for tests and their timeout in the patch
checkTests () {
echo
echo
@@ -357,6 +357,24 @@ checkTests () {
JIRA_COMMENT=$JIRA_COMMENT
{color:green}+1 tests included{color}. The patch appears to include
$testReferences new or modified test files.
+ echo
+ echo ==
+ echo ==
+ echo Checking if the tests have timeout assigned in this patch.
+ echo ==
+ echo ==
+
+ nontimeoutTests=`cat $PATCH_DIR/patch | $AWK '{ printf %s , $0 }' | $GREP
--extended-regex --count '[ ]*\+[ ]*((@Test[\+ ]*[A-Za-z]+)|([\+ ]*@Test[
\+]*\([ \+]*\)[\ ]*\+?[ ]*[A-Za-z]+)|([\+ ]*@Test[\+ ]*\(exception[ \+]*=[
\+]*[A-Z.a-z0-9A-Z ]*\)))'`
+
+ if [[ $nontimeoutTests == 0 ]] ; then
+JIRA_COMMENT=$JIRA_COMMENT
+
+{color:green}+1 tests included appear to have a timeout.{color}
+ return 1
+ fi
+ JIRA_COMMENT=$JIRA_COMMENT
+
+ {color:red}-1 one of tests included doesn't have a timeout.{color}
return 0
}
svn commit: r1448285 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Wed Feb 20 16:41:57 2013 New Revision: 1448285 URL: http://svn.apache.org/r1448285 Log: HADOOP-9112. test-patch should -1 for @Tests without a timeout (Surenkumar Nihalani via bobby) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1448285r1=1448284r2=1448285view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Feb 20 16:41:57 2013 @@ -150,6 +150,9 @@ Trunk (Unreleased) HADOOP-9218 Document the Rpc-wrappers used internally (sanjay Radia) +HADOOP-9112. test-patch should -1 for @Tests without a timeout +(Surenkumar Nihalani via bobby) + BUG FIXES HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang)
svn commit: r1442174 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
Author: bobby
Date: Mon Feb 4 16:54:43 2013
New Revision: 1442174
URL: http://svn.apache.org/viewvc?rev=1442174view=rev
Log:
svn merge -c r1427945 FIXES: MAPREDUCE-4884. Streaming tests fail to start
MiniMRCluster due to missing queue configuration. Contributed by Chris Nauroth.
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml?rev=1442174r1=1442173r2=1442174view=diff
==
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml
(original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml Mon
Feb 4 16:54:43 2013
@@ -141,5 +141,14 @@
/configuration
/plugin
/plugins
+testResources
+ testResource
+
directory${basedir}/../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/directory
+includes
+ includecapacity-scheduler.xml/include
+/includes
+filteringfalse/filtering
+ /testResource
+/testResources
/build
/project
svn commit: r1441635 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Author: bobby
Date: Fri Feb 1 21:17:35 2013
New Revision: 1441635
URL: http://svn.apache.org/viewvc?rev=1441635view=rev
Log:
svn merge -c 1441628 FIXES: HADOOP-9067. provide test for
LocalFileSystem.reportChecksumFailure (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1441635r1=1441634r2=1441635view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Feb 1 21:17:35 2013
@@ -978,6 +978,9 @@ Release 0.23.7 - UNRELEASED
HADOOP-8849. FileUtil#fullyDelete should grant the target directories +rwx
permissions (Ivan A. Veselovsky via bobby)
+HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure
+(Ivan A. Veselovsky via bobby)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1441635r1=1441634r2=1441635view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Fri Feb 1 21:17:35 2013
@@ -26,6 +26,7 @@ import java.io.*;
import static org.junit.Assert.*;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -36,8 +37,9 @@ public class TestLocalFileSystem {
private static final String TEST_ROOT_DIR
= System.getProperty(test.build.data,build/test/data) +
/work-dir/localfs;
+ private final File base = new File(TEST_ROOT_DIR);
private Configuration conf;
- private FileSystem fileSys;
+ private LocalFileSystem fileSys;
private void cleanupFile(FileSystem fs, Path name) throws IOException {
assertTrue(fs.exists(name));
@@ -51,6 +53,13 @@ public class TestLocalFileSystem {
fileSys = FileSystem.getLocal(conf);
fileSys.delete(new Path(TEST_ROOT_DIR), true);
}
+
+ @After
+ public void after() throws IOException {
+base.setWritable(true);
+FileUtil.fullyDelete(base);
+assertTrue(!base.exists());
+ }
/**
* Test the capability of setting the working directory.
@@ -266,10 +275,83 @@ public class TestLocalFileSystem {
LocalFileSystem fs = FileSystem.getLocal(conf);
File colonFile = new File(TEST_ROOT_DIR, foo:bar);
colonFile.mkdirs();
-colonFile.createNewFile();
FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
assertEquals(Unexpected number of stats, 1, stats.length);
assertEquals(Bad path from stat, colonFile.getAbsolutePath(),
stats[0].getPath().toUri().getPath());
}
+
+ @Test
+ public void testReportChecksumFailure() throws IOException {
+base.mkdirs();
+assertTrue(base.exists() base.isDirectory());
+
+final File dir1 = new File(base, dir1);
+final File dir2 = new File(dir1, dir2);
+dir2.mkdirs();
+assertTrue(dir2.exists() dir2.canWrite());
+
+final String dataFileName = corruptedData;
+final Path dataPath = new Path(new File(dir2, dataFileName).toURI());
+final Path checksumPath = fileSys.getChecksumFile(dataPath);
+final FSDataOutputStream fsdos = fileSys.create(dataPath);
+try {
+ fsdos.writeUTF(foo);
+} finally {
+ fsdos.close();
+}
+assertTrue(fileSys.pathToFile(dataPath).exists());
+final long dataFileLength = fileSys.getFileStatus(dataPath).getLen();
+assertTrue(dataFileLength 0);
+
+// check the the checksum file is created and not empty:
+assertTrue(fileSys.pathToFile(checksumPath).exists());
+final long checksumFileLength =
fileSys.getFileStatus(checksumPath).getLen();
+assertTrue(checksumFileLength 0);
+
+// this is a hack to force the #reportChecksumFailure() method to stop
+// climbing up at the 'base' directory and use 'dir1/bad_files' as the
+// corrupted files storage:
+base.setWritable(false);
+
+FSDataInputStream dataFsdis = fileSys.open(dataPath);
+FSDataInputStream checksumFsdis = fileSys.open(checksumPath);
+
+boolean retryIsNecessary
svn commit: r1441638 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Author: bobby
Date: Fri Feb 1 21:25:15 2013
New Revision: 1441638
URL: http://svn.apache.org/viewvc?rev=1441638view=rev
Log:
svn merge -c 1441628 FIXES: HADOOP-9067. provide test for
LocalFileSystem.reportChecksumFailure (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1441638r1=1441637r2=1441638view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Feb 1 21:25:15 2013
@@ -14,6 +14,8 @@ Release 0.23.7 - UNRELEASED
HADOOP-9216. CompressionCodecFactory#getCodecClasses should trim the
result of parsing by Configuration. (Tsuyoshi Ozawa via todd)
+HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure
+(Ivan A. Veselovsky via bobby)
OPTIMIZATIONS
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1441638r1=1441637r2=1441638view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Fri Feb 1 21:25:15 2013
@@ -26,6 +26,7 @@ import java.io.*;
import static org.junit.Assert.*;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -36,8 +37,9 @@ public class TestLocalFileSystem {
private static String TEST_ROOT_DIR
= System.getProperty(test.build.data,build/test/data/work-dir/localfs);
+ private final File base = new File(TEST_ROOT_DIR);
private Configuration conf;
- private FileSystem fileSys;
+ private LocalFileSystem fileSys;
private void cleanupFile(FileSystem fs, Path name) throws IOException {
assertTrue(fs.exists(name));
@@ -51,6 +53,13 @@ public class TestLocalFileSystem {
fileSys = FileSystem.getLocal(conf);
fileSys.delete(new Path(TEST_ROOT_DIR), true);
}
+
+ @After
+ public void after() throws IOException {
+base.setWritable(true);
+FileUtil.fullyDelete(base);
+assertTrue(!base.exists());
+ }
/**
* Test the capability of setting the working directory.
@@ -255,10 +264,83 @@ public class TestLocalFileSystem {
LocalFileSystem fs = FileSystem.getLocal(conf);
File colonFile = new File(TEST_ROOT_DIR, foo:bar);
colonFile.mkdirs();
-colonFile.createNewFile();
FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
assertEquals(Unexpected number of stats, 1, stats.length);
assertEquals(Bad path from stat, colonFile.getAbsolutePath(),
stats[0].getPath().toUri().getPath());
}
+
+ @Test
+ public void testReportChecksumFailure() throws IOException {
+base.mkdirs();
+assertTrue(base.exists() base.isDirectory());
+
+final File dir1 = new File(base, dir1);
+final File dir2 = new File(dir1, dir2);
+dir2.mkdirs();
+assertTrue(dir2.exists() dir2.canWrite());
+
+final String dataFileName = corruptedData;
+final Path dataPath = new Path(new File(dir2, dataFileName).toURI());
+final Path checksumPath = fileSys.getChecksumFile(dataPath);
+final FSDataOutputStream fsdos = fileSys.create(dataPath);
+try {
+ fsdos.writeUTF(foo);
+} finally {
+ fsdos.close();
+}
+assertTrue(fileSys.pathToFile(dataPath).exists());
+final long dataFileLength = fileSys.getFileStatus(dataPath).getLen();
+assertTrue(dataFileLength 0);
+
+// check the the checksum file is created and not empty:
+assertTrue(fileSys.pathToFile(checksumPath).exists());
+final long checksumFileLength =
fileSys.getFileStatus(checksumPath).getLen();
+assertTrue(checksumFileLength 0);
+
+// this is a hack to force the #reportChecksumFailure() method to stop
+// climbing up at the 'base' directory and use 'dir1/bad_files' as the
+// corrupted files storage:
+base.setWritable(false);
+
+FSDataInputStream dataFsdis = fileSys.open(dataPath);
+FSDataInputStream checksumFsdis = fileSys.open(checksumPath);
+
+boolean
svn commit: r1434868 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/FileUtil.java src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Author: bobby
Date: Thu Jan 17 19:19:10 2013
New Revision: 1434868
URL: http://svn.apache.org/viewvc?rev=1434868view=rev
Log:
HADOOP-8849. FileUtil#fullyDelete should grant the target directories +rwx
permissions (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1434868r1=1434867r2=1434868view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Jan
17 19:19:10 2013
@@ -1264,6 +1264,9 @@ Release 0.23.7 - UNRELEASED
IMPROVEMENTS
+HADOOP-8849. FileUtil#fullyDelete should grant the target directories +rwx
+permissions (Ivan A. Veselovsky via bobby)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1434868r1=1434867r2=1434868view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
Thu Jan 17 19:19:10 2013
@@ -87,33 +87,98 @@ public class FileUtil {
* (4) If dir is a normal directory, then dir and all its contents
recursively
* are deleted.
*/
- public static boolean fullyDelete(File dir) {
-if (dir.delete()) {
+ public static boolean fullyDelete(final File dir) {
+return fullyDelete(dir, false);
+ }
+
+ /**
+ * Delete a directory and all its contents. If
+ * we return false, the directory may be partially-deleted.
+ * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+ * to by the symlink is not deleted.
+ * (2) If dir is symlink to a directory, symlink is deleted. The directory
+ * pointed to by symlink is not deleted.
+ * (3) If dir is a normal file, it is deleted.
+ * (4) If dir is a normal directory, then dir and all its contents
recursively
+ * are deleted.
+ * @param dir the file or directory to be deleted
+ * @param tryGrantPermissions true if permissions should be modified to
delete a file.
+ * @return true on success false on failure.
+ */
+ public static boolean fullyDelete(final File dir, boolean
tryGrantPermissions) {
+if (tryGrantPermissions) {
+ // try to chmod +rwx the parent folder of the 'dir':
+ File parent = dir.getParentFile();
+ grantPermissions(parent);
+}
+if (deleteImpl(dir, false)) {
// dir is (a) normal file, (b) symlink to a file, (c) empty directory or
// (d) symlink to a directory
return true;
}
-
// handle nonempty directory deletion
-if (!fullyDeleteContents(dir)) {
+if (!fullyDeleteContents(dir, tryGrantPermissions)) {
return false;
}
-return dir.delete();
+return deleteImpl(dir, true);
+ }
+
+ /*
+ * Pure-Java implementation of chmod +rwx f.
+ */
+ private static void grantPermissions(final File f) {
+ f.setExecutable(true);
+ f.setReadable(true);
+ f.setWritable(true);
}
+ private static boolean deleteImpl(final File f, final boolean doLog) {
+if (f == null) {
+ LOG.warn(null file argument.);
+ return false;
+}
+final boolean wasDeleted = f.delete();
+if (wasDeleted) {
+ return true;
+}
+final boolean ex = f.exists();
+if (doLog ex) {
+ LOG.warn(Failed to delete file or dir [
+ + f.getAbsolutePath() + ]: it still exists.);
+}
+return !ex;
+ }
+
+ /**
+ * Delete the contents of a directory, not the directory itself. If
+ * we return false, the directory may be partially-deleted.
+ * If dir is a symlink to a directory, all the contents of the actual
+ * directory pointed to by dir will be deleted.
+ */
+ public static boolean fullyDeleteContents(final File dir) {
+return fullyDeleteContents(dir, false);
+ }
+
/**
* Delete the contents of a directory, not the directory itself. If
* we return false, the directory may be partially-deleted.
* If dir is a symlink to a directory, all the contents of the actual
* directory pointed to by dir will be deleted.
+ * @param tryGrantPermissions
svn commit: r1434870 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/FileUtil.java src/test/java/org/apache/hadoop/fs/TestFile
Author: bobby
Date: Thu Jan 17 19:23:13 2013
New Revision: 1434870
URL: http://svn.apache.org/viewvc?rev=1434870view=rev
Log:
svn merge -c 1434868 FIXES: HADOOP-8849. FileUtil#fullyDelete should grant the
target directories +rwx permissions (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1434870r1=1434869r2=1434870view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Jan 17 19:23:13 2013
@@ -957,6 +957,9 @@ Release 0.23.7 - UNRELEASED
IMPROVEMENTS
+HADOOP-8849. FileUtil#fullyDelete should grant the target directories +rwx
+permissions (Ivan A. Veselovsky via bobby)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1434870r1=1434869r2=1434870view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
Thu Jan 17 19:23:13 2013
@@ -87,33 +87,98 @@ public class FileUtil {
* (4) If dir is a normal directory, then dir and all its contents
recursively
* are deleted.
*/
- public static boolean fullyDelete(File dir) {
-if (dir.delete()) {
+ public static boolean fullyDelete(final File dir) {
+return fullyDelete(dir, false);
+ }
+
+ /**
+ * Delete a directory and all its contents. If
+ * we return false, the directory may be partially-deleted.
+ * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+ * to by the symlink is not deleted.
+ * (2) If dir is symlink to a directory, symlink is deleted. The directory
+ * pointed to by symlink is not deleted.
+ * (3) If dir is a normal file, it is deleted.
+ * (4) If dir is a normal directory, then dir and all its contents
recursively
+ * are deleted.
+ * @param dir the file or directory to be deleted
+ * @param tryGrantPermissions true if permissions should be modified to
delete a file.
+ * @return true on success false on failure.
+ */
+ public static boolean fullyDelete(final File dir, boolean
tryGrantPermissions) {
+if (tryGrantPermissions) {
+ // try to chmod +rwx the parent folder of the 'dir':
+ File parent = dir.getParentFile();
+ grantPermissions(parent);
+}
+if (deleteImpl(dir, false)) {
// dir is (a) normal file, (b) symlink to a file, (c) empty directory or
// (d) symlink to a directory
return true;
}
-
// handle nonempty directory deletion
-if (!fullyDeleteContents(dir)) {
+if (!fullyDeleteContents(dir, tryGrantPermissions)) {
return false;
}
-return dir.delete();
+return deleteImpl(dir, true);
+ }
+
+ /*
+ * Pure-Java implementation of chmod +rwx f.
+ */
+ private static void grantPermissions(final File f) {
+ f.setExecutable(true);
+ f.setReadable(true);
+ f.setWritable(true);
}
+ private static boolean deleteImpl(final File f, final boolean doLog) {
+if (f == null) {
+ LOG.warn(null file argument.);
+ return false;
+}
+final boolean wasDeleted = f.delete();
+if (wasDeleted) {
+ return true;
+}
+final boolean ex = f.exists();
+if (doLog ex) {
+ LOG.warn(Failed to delete file or dir [
+ + f.getAbsolutePath() + ]: it still exists.);
+}
+return !ex;
+ }
+
+ /**
+ * Delete the contents of a directory, not the directory itself. If
+ * we return false, the directory may be partially-deleted.
+ * If dir is a symlink to a directory, all the contents of the actual
+ * directory pointed to by dir will be deleted.
+ */
+ public static boolean fullyDeleteContents(final File dir) {
+return fullyDeleteContents(dir, false);
+ }
+
/**
* Delete the contents of a directory, not the directory itself. If
* we return false, the directory may be partially-deleted
svn commit: r1432949 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Mon Jan 14 15:14:39 2013 New Revision: 1432949 URL: http://svn.apache.org/viewvc?rev=1432949view=rev Log: HADOOP-9202. test-patch.sh fails during mvn eclipse:eclipse if patch adds a new module to the build (Chris Nauroth via bobby) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1432949r1=1432948r2=1432949view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Jan 14 15:14:39 2013 @@ -308,10 +308,13 @@ Trunk (Unreleased) HADOOP-9131. Turn off TestLocalFileSystem#testListStatusWithColons on Windows. (Chris Nauroth via suresh) -HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for +HADOOP-8957. AbstractFileSystem#IsValidName should be overridden for embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia) -HADOOP-9139 improve killKdc.sh (Ivan A. Veselovsky via bobby) +HADOOP-9139. improve killKdc.sh (Ivan A. Veselovsky via bobby) + +HADOOP-9202. test-patch.sh fails during mvn eclipse:eclipse if patch adds +a new module to the build (Chris Nauroth via bobby) OPTIMIZATIONS
svn commit: r1432949 - /hadoop/common/trunk/dev-support/test-patch.sh
Author: bobby Date: Mon Jan 14 15:14:39 2013 New Revision: 1432949 URL: http://svn.apache.org/viewvc?rev=1432949view=rev Log: HADOOP-9202. test-patch.sh fails during mvn eclipse:eclipse if patch adds a new module to the build (Chris Nauroth via bobby) Modified: hadoop/common/trunk/dev-support/test-patch.sh Modified: hadoop/common/trunk/dev-support/test-patch.sh URL: http://svn.apache.org/viewvc/hadoop/common/trunk/dev-support/test-patch.sh?rev=1432949r1=1432948r2=1432949view=diff == --- hadoop/common/trunk/dev-support/test-patch.sh (original) +++ hadoop/common/trunk/dev-support/test-patch.sh Mon Jan 14 15:14:39 2013 @@ -980,12 +980,12 @@ fi (( RESULT = RESULT + $JAVAC_RET )) checkJavadocWarnings (( RESULT = RESULT + $? )) -checkEclipseGeneration -(( RESULT = RESULT + $? )) ### Checkstyle not implemented yet #checkStyle #(( RESULT = RESULT + $? )) buildAndInstall +checkEclipseGeneration +(( RESULT = RESULT + $? )) checkFindbugsWarnings (( RESULT = RESULT + $? )) checkReleaseAuditWarnings
svn commit: r1432151 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/resources/kdc/killKdc.sh
Author: bobby
Date: Fri Jan 11 16:59:32 2013
New Revision: 1432151
URL: http://svn.apache.org/viewvc?rev=1432151view=rev
Log:
HADOOP-9139 improve killKdc.sh (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1432151r1=1432150r2=1432151view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Jan
11 16:59:32 2013
@@ -311,6 +311,8 @@ Trunk (Unreleased)
HADOOP-8957 AbstractFileSystem#IsValidName should be overridden for
embedded file systems like ViewFs (Chris Nauroth via Sanjay Radia)
+HADOOP-9139 improve killKdc.sh (Ivan A. Veselovsky via bobby)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh?rev=1432151r1=1432150r2=1432151view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh
Fri Jan 11 16:59:32 2013
@@ -1,3 +1,19 @@
#!/bin/sh
-ps -ef | grep apacheds | grep -v grep | cut -f4 -d ' ' |xargs kill -9
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# License); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ps -ef | grep apacheds | grep -v grep | awk '{printf $2\n}' | xargs -t
--no-run-if-empty kill -9
svn commit: r1432243 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Author: bobby
Date: Fri Jan 11 19:26:29 2013
New Revision: 1432243
URL: http://svn.apache.org/viewvc?rev=1432243view=rev
Log:
MAPREDUCE-4933. MR1 final merge asks for length of file it just wrote before
flushing it. Contributed by Sandy Ryza
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1432243r1=1432242r2=1432243view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Jan 11 19:26:29 2013
@@ -425,6 +425,9 @@ Release 1.2.0 - unreleased
MAPREDUCE-4850. Job recovery may fail if staging directory has been
deleted. (tomwhite)
+MAPREDUCE-4933. MR1 final merge asks for length of file it just wrote
+before flushing it. (Sandy Ryza via bobby)
+
Release 1.1.2 - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1432243r1=1432242r2=1432243view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Fri Jan 11 19:26:29 2013
@@ -2465,10 +2465,12 @@ class ReduceTask extends Task {
final RawKeyValueIterator rIter = Merger.merge(job, fs,
keyClass, valueClass, memDiskSegments, numMemDiskSegments,
tmpDir, comparator, reporter, spilledRecordsCounter, null);
- final Writer writer = new Writer(job, fs, outputPath,
+ Writer writer = new Writer(job, fs, outputPath,
keyClass, valueClass, codec, null);
try {
Merger.writeFile(rIter, writer, reporter, job);
+writer.close();
+writer = null;
addToMapOutputFilesOnDisk(fs.getFileStatus(outputPath));
} catch (Exception e) {
if (null != outputPath) {
svn commit: r1424566 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/
Author: bobby
Date: Thu Dec 20 16:19:48 2012
New Revision: 1424566
URL: http://svn.apache.org/viewvc?rev=1424566view=rev
Log:
HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1424566r1=1424565r2=1424566view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Dec
20 16:19:48 2012
@@ -1224,6 +1224,8 @@ Release 0.23.6 - UNRELEASED
HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
Veselovsky via bobby)
+HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java?rev=1424566r1=1424565r2=1424566view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
Thu Dec 20 16:19:48 2012
@@ -311,6 +311,7 @@ abstract public class Command extends Co
if (recursive item.stat.isDirectory()) {
recursePath(item);
}
+postProcessPath(item);
} catch (IOException e) {
displayError(e);
}
@@ -330,6 +331,15 @@ abstract public class Command extends Co
}
/**
+ * Hook for commands to implement an operation to be applied on each
+ * path for the command after being processed successfully
+ * @param item a {@link PathData} object
+ * @throws IOException if anything goes wrong...
+ */
+ protected void postProcessPath(PathData item) throws IOException {
+ }
+
+ /**
* Gets the directory listing for a path and invokes
* {@link #processPaths(PathData, PathData...)}
* @param item {@link PathData} for directory to recurse into
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java?rev=1424566r1=1424565r2=1424566view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
Thu Dec 20 16:19:48 2012
@@ -24,6 +24,7 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathExistsException;
import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
/** Various commands for moving files */
@@ -49,7 +50,21 @@ class MoveCommands {
@Override
protected void processPath(PathData src, PathData target) throws
IOException {
- target.fs.moveFromLocalFile(src.path, target.path);
+ // unlike copy, don't merge existing dirs during move
+ if (target.exists target.stat.isDirectory()) {
+throw new PathExistsException(target.toString());
+ }
+ super.processPath(src, target);
+}
+
+@Override
+protected void postProcessPath(PathData src) throws IOException {
+ if (!src.fs.delete(src.path, false)) {
+// we have no way to know the actual error...
+PathIOException e = new PathIOException(src.toString());
+e.setOperation(remove);
+throw e;
+ }
}
}
@@ -95,4 +110,4 @@ class MoveCommands {
}
}
}
-}
\ No newline at end of file
+}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project
svn commit: r1424574 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/
Author: bobby
Date: Thu Dec 20 16:30:20 2012
New Revision: 1424574
URL: http://svn.apache.org/viewvc?rev=1424574view=rev
Log:
svn merge -c 1424566 FIXES: HADOOP-9105. FsShell -moveFromLocal erroneously
fails (daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1424574r1=1424573r2=1424574view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Dec 20 16:30:20 2012
@@ -919,6 +919,8 @@ Release 0.23.6 - UNRELEASED
HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
Veselovsky via bobby)
+HADOOP-9105. FsShell -moveFromLocal erroneously fails (daryn via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java?rev=1424574r1=1424573r2=1424574view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
Thu Dec 20 16:30:20 2012
@@ -307,6 +307,7 @@ abstract public class Command extends Co
if (recursive item.stat.isDirectory()) {
recursePath(item);
}
+postProcessPath(item);
} catch (IOException e) {
displayError(e);
}
@@ -326,6 +327,15 @@ abstract public class Command extends Co
}
/**
+ * Hook for commands to implement an operation to be applied on each
+ * path for the command after being processed successfully
+ * @param item a {@link PathData} object
+ * @throws IOException if anything goes wrong...
+ */
+ protected void postProcessPath(PathData item) throws IOException {
+ }
+
+ /**
* Gets the directory listing for a path and invokes
* {@link #processPaths(PathData, PathData...)}
* @param item {@link PathData} for directory to recurse into
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java?rev=1424574r1=1424573r2=1424574view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
Thu Dec 20 16:30:20 2012
@@ -24,6 +24,7 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
+import org.apache.hadoop.fs.PathExistsException;
import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
/** Various commands for moving files */
@@ -49,7 +50,21 @@ class MoveCommands {
@Override
protected void processPath(PathData src, PathData target) throws
IOException {
- target.fs.moveFromLocalFile(src.path, target.path);
+ // unlike copy, don't merge existing dirs during move
+ if (target.exists target.stat.isDirectory()) {
+throw new PathExistsException(target.toString());
+ }
+ super.processPath(src, target);
+}
+
+@Override
+protected void postProcessPath(PathData src) throws IOException {
+ if (!src.fs.delete(src.path, false)) {
+// we have no way to know the actual error...
+PathIOException e = new PathIOException(src.toString());
+e.setOperation(remove);
+throw e;
+ }
}
}
@@ -95,4 +110,4 @@ class MoveCommands
svn commit: r1414239 - in /hadoop/common/trunk/hadoop-tools: hadoop-archives/src/test/java/org/apache/hadoop/tools/ hadoop-distcp/src/test/java/org/apache/hadoop/tools/ hadoop-extras/src/test/java/org
Author: bobby
Date: Tue Nov 27 16:05:16 2012
New Revision: 1414239
URL: http://svn.apache.org/viewvc?rev=1414239view=rev
Log:
YARN-204. test coverage for org.apache.hadoop.tools (Aleksey Gorshkov via bobby)
Added:
hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java?rev=1414239r1=1414238r2=1414239view=diff
==
---
hadoop/common/trunk/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
(original)
+++
hadoop/common/trunk/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
Tue Nov 27 16:05:16 2012
@@ -27,8 +27,6 @@ import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
@@ -38,111 +36,117 @@ import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.ToolRunner;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* test {@link HadoopArchives}
*/
-public class TestHadoopArchives extends TestCase {
+public class TestHadoopArchives {
- public static final String HADOOP_ARCHIVES_JAR =
JarFinder.getJar(HadoopArchives.class);
+ public static final String HADOOP_ARCHIVES_JAR = JarFinder
+ .getJar(HadoopArchives.class);
{
-((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class)
-).getLogger().setLevel(Level.ERROR);
-((Log4JLogger)org.apache.hadoop.ipc.Server.LOG
-).getLogger().setLevel(Level.ERROR);
-((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG
-).getLogger().setLevel(Level.ERROR);
+((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class))
+.getLogger().setLevel(Level.ERROR);
+
}
private static final String inputDir = input;
private Path inputPath;
private MiniDFSCluster dfscluster;
- private MiniMRCluster mapred;
+
+ private Configuration conf;
private FileSystem fs;
private Path archivePath;
-
- static private Path createFile(Path dir, String filename, FileSystem fs
- ) throws IOException {
+
+ static private Path createFile(Path dir, String filename, FileSystem fs)
+ throws IOException {
final Path f = new Path(dir, filename);
-final FSDataOutputStream out = fs.create(f);
+final FSDataOutputStream out = fs.create(f);
out.write(filename.getBytes());
out.close();
return f;
}
-
- protected void setUp() throws Exception {
-super.setUp();
-dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null);
+
+ @Before
+ public void setUp() throws Exception {
+conf = new Configuration();
+conf.set(CapacitySchedulerConfiguration.PREFIX
++ CapacitySchedulerConfiguration.ROOT + .
++ CapacitySchedulerConfiguration.QUEUES, default);
+conf.set(CapacitySchedulerConfiguration.PREFIX
++ CapacitySchedulerConfiguration.ROOT + .default.
++ CapacitySchedulerConfiguration.CAPACITY, 100);
+dfscluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true)
+.build();
+
fs = dfscluster.getFileSystem();
-mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
-inputPath = new Path(fs.getHomeDirectory(), inputDir);
+inputPath = new Path(fs.getHomeDirectory(), inputDir);
archivePath = new Path(fs.getHomeDirectory(), archive);
fs.mkdirs(inputPath);
createFile(inputPath, a, fs);
createFile(inputPath, b, fs);
createFile(inputPath, c, fs);
}
-
- protected void tearDown() throws Exception {
+
+ @After
+ public void tearDown() throws Exception {
try {
- if (mapred != null
svn commit: r1414241 - in /hadoop/common/branches/branch-2/hadoop-tools: hadoop-archives/src/test/java/org/apache/hadoop/tools/ hadoop-distcp/src/test/java/org/apache/hadoop/tools/ hadoop-extras/src/t
Author: bobby
Date: Tue Nov 27 16:06:57 2012
New Revision: 1414241
URL: http://svn.apache.org/viewvc?rev=1414241view=rev
Log:
svn merge -c 1414239 FIXES: YARN-204. test coverage for org.apache.hadoop.tools
(Aleksey Gorshkov via bobby)
Added:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
- copied unchanged from r1414239,
hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
- copied unchanged from r1414239,
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java?rev=1414241r1=1414240r2=1414241view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
Tue Nov 27 16:06:57 2012
@@ -27,8 +27,6 @@ import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
@@ -38,111 +36,117 @@ import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.ToolRunner;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* test {@link HadoopArchives}
*/
-public class TestHadoopArchives extends TestCase {
+public class TestHadoopArchives {
- public static final String HADOOP_ARCHIVES_JAR =
JarFinder.getJar(HadoopArchives.class);
+ public static final String HADOOP_ARCHIVES_JAR = JarFinder
+ .getJar(HadoopArchives.class);
{
-((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class)
-).getLogger().setLevel(Level.ERROR);
-((Log4JLogger)org.apache.hadoop.ipc.Server.LOG
-).getLogger().setLevel(Level.ERROR);
-((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG
-).getLogger().setLevel(Level.ERROR);
+((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class))
+.getLogger().setLevel(Level.ERROR);
+
}
private static final String inputDir = input;
private Path inputPath;
private MiniDFSCluster dfscluster;
- private MiniMRCluster mapred;
+
+ private Configuration conf;
private FileSystem fs;
private Path archivePath;
-
- static private Path createFile(Path dir, String filename, FileSystem fs
- ) throws IOException {
+
+ static private Path createFile(Path dir, String filename, FileSystem fs)
+ throws IOException {
final Path f = new Path(dir, filename);
-final FSDataOutputStream out = fs.create(f);
+final FSDataOutputStream out = fs.create(f);
out.write(filename.getBytes());
out.close();
return f;
}
-
- protected void setUp() throws Exception {
-super.setUp();
-dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null);
+
+ @Before
+ public void setUp() throws Exception {
+conf = new Configuration();
+conf.set(CapacitySchedulerConfiguration.PREFIX
++ CapacitySchedulerConfiguration.ROOT + .
++ CapacitySchedulerConfiguration.QUEUES, default);
+conf.set(CapacitySchedulerConfiguration.PREFIX
++ CapacitySchedulerConfiguration.ROOT + .default.
++ CapacitySchedulerConfiguration.CAPACITY, 100);
+dfscluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true)
+.build();
+
fs = dfscluster.getFileSystem();
-mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
-inputPath = new Path(fs.getHomeDirectory
svn commit: r1414243 - in /hadoop/common/branches/branch-0.23/hadoop-tools: hadoop-archives/src/test/java/org/apache/hadoop/tools/ hadoop-distcp/src/test/java/org/apache/hadoop/tools/ hadoop-extras/sr
Author: bobby
Date: Tue Nov 27 16:12:33 2012
New Revision: 1414243
URL: http://svn.apache.org/viewvc?rev=1414243view=rev
Log:
svn merge -c 1414239 FIXES: YARN-204. test coverage for org.apache.hadoop.tools
(Aleksey Gorshkov via bobby)
Added:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
- copied unchanged from r1414239,
hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
- copied unchanged from r1414239,
hadoop/common/trunk/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java?rev=1414243r1=1414242r2=1414243view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
Tue Nov 27 16:12:33 2012
@@ -27,8 +27,6 @@ import java.util.Collections;
import java.util.List;
import java.util.StringTokenizer;
-import junit.framework.TestCase;
-
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
@@ -38,111 +36,117 @@ import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.ToolRunner;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
/**
* test {@link HadoopArchives}
*/
-public class TestHadoopArchives extends TestCase {
+public class TestHadoopArchives {
- public static final String HADOOP_ARCHIVES_JAR =
JarFinder.getJar(HadoopArchives.class);
+ public static final String HADOOP_ARCHIVES_JAR = JarFinder
+ .getJar(HadoopArchives.class);
{
-((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class)
-).getLogger().setLevel(Level.ERROR);
-((Log4JLogger)org.apache.hadoop.ipc.Server.LOG
-).getLogger().setLevel(Level.ERROR);
-((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG
-).getLogger().setLevel(Level.ERROR);
+((Log4JLogger) LogFactory.getLog(org.apache.hadoop.security.Groups.class))
+.getLogger().setLevel(Level.ERROR);
+
}
private static final String inputDir = input;
private Path inputPath;
private MiniDFSCluster dfscluster;
- private MiniMRCluster mapred;
+
+ private Configuration conf;
private FileSystem fs;
private Path archivePath;
-
- static private Path createFile(Path dir, String filename, FileSystem fs
- ) throws IOException {
+
+ static private Path createFile(Path dir, String filename, FileSystem fs)
+ throws IOException {
final Path f = new Path(dir, filename);
-final FSDataOutputStream out = fs.create(f);
+final FSDataOutputStream out = fs.create(f);
out.write(filename.getBytes());
out.close();
return f;
}
-
- protected void setUp() throws Exception {
-super.setUp();
-dfscluster = new MiniDFSCluster(new Configuration(), 2, true, null);
+
+ @Before
+ public void setUp() throws Exception {
+conf = new Configuration();
+conf.set(CapacitySchedulerConfiguration.PREFIX
++ CapacitySchedulerConfiguration.ROOT + .
++ CapacitySchedulerConfiguration.QUEUES, default);
+conf.set(CapacitySchedulerConfiguration.PREFIX
++ CapacitySchedulerConfiguration.ROOT + .default.
++ CapacitySchedulerConfiguration.CAPACITY, 100);
+dfscluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true)
+.build();
+
fs = dfscluster.getFileSystem();
-mapred = new MiniMRCluster(2, fs.getUri().toString(), 1);
-inputPath = new Path
svn commit: r1413743 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/HarFileSystem.java src/test/java/org/apache/hadoop/fs/TestHarFileSyst
Author: bobby
Date: Mon Nov 26 17:45:57 2012
New Revision: 1413743
URL: http://svn.apache.org/viewvc?rev=1413743view=rev
Log:
HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
Veselovsky via bobby)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1413743r1=1413742r2=1413743view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov
26 17:45:57 2012
@@ -1152,6 +1152,9 @@ Release 0.23.6 - UNRELEASED
HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins
(Robert Parker via tgraves)
+HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
+Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1413743r1=1413742r2=1413743view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Mon Nov 26 17:45:57 2012
@@ -584,13 +584,6 @@ public class HarFileSystem extends Filte
public String getName() {
return name;
}
-
-public ListString getChildren() {
- return children;
-}
-public String getFileName() {
- return name;
-}
public String getPartName() {
return partName;
}
@@ -662,15 +655,6 @@ public class HarFileSystem extends Filte
hstatus.getStartIndex(), hstatus.getLength(), bufferSize);
}
- /*
- * create throws an exception in Har filesystem.
- * The archive once created cannot be changed.
- */
- public FSDataOutputStream create(Path f, int bufferSize)
-throws IOException {
-throw new IOException(Har: Create not allowed);
- }
-
@Override
public FSDataOutputStream create(Path f,
FsPermission permission,
@@ -1106,4 +1090,11 @@ public class HarFileSystem extends Filte
}
}
}
+
+ /*
+ * testing purposes only:
+ */
+ HarMetaData getMetadata() {
+return metadata;
+ }
}
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java?rev=1413743view=auto
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Mon Nov 26 17:45:57 2012
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import
svn commit: r1413744 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/HarFileSystem.java src/test/java/org/apache/hadoop/fs/Tes
Author: bobby
Date: Mon Nov 26 17:46:55 2012
New Revision: 1413744
URL: http://svn.apache.org/viewvc?rev=1413744view=rev
Log:
svn merge -c 1413743 FIXES: HADOOP-8992. Enhance unit-test coverage of class
HarFileSystem (Ivan A. Veselovsky via bobby)
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
- copied unchanged from r1413743,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1413744r1=1413743r2=1413744view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Nov 26 17:46:55 2012
@@ -860,6 +860,9 @@ Release 0.23.6 - UNRELEASED
HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins
(Robert Parker via tgraves)
+HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
+Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1413744r1=1413743r2=1413744view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Mon Nov 26 17:46:55 2012
@@ -584,13 +584,6 @@ public class HarFileSystem extends Filte
public String getName() {
return name;
}
-
-public ListString getChildren() {
- return children;
-}
-public String getFileName() {
- return name;
-}
public String getPartName() {
return partName;
}
@@ -662,15 +655,6 @@ public class HarFileSystem extends Filte
hstatus.getStartIndex(), hstatus.getLength(), bufferSize);
}
- /*
- * create throws an exception in Har filesystem.
- * The archive once created cannot be changed.
- */
- public FSDataOutputStream create(Path f, int bufferSize)
-throws IOException {
-throw new IOException(Har: Create not allowed);
- }
-
@Override
public FSDataOutputStream create(Path f,
FsPermission permission,
@@ -1106,4 +1090,11 @@ public class HarFileSystem extends Filte
}
}
}
+
+ /*
+ * testing purposes only:
+ */
+ HarMetaData getMetadata() {
+return metadata;
+ }
}
svn commit: r1413746 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/HarFileSystem.java src/test/java/org/apache/hadoop/fs/
Author: bobby
Date: Mon Nov 26 17:49:57 2012
New Revision: 1413746
URL: http://svn.apache.org/viewvc?rev=1413746view=rev
Log:
svn merge -c 1413743 FIXES: HADOOP-8992. Enhance unit-test coverage of class
HarFileSystem (Ivan A. Veselovsky via bobby)
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
- copied unchanged from r1413743,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystemBasics.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1413746r1=1413745r2=1413746view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Nov 26 17:49:57 2012
@@ -17,6 +17,9 @@ Release 0.23.6 - UNRELEASED
HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins
(Robert Parker via tgraves)
+HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
+Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java?rev=1413746r1=1413745r2=1413746view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
Mon Nov 26 17:49:57 2012
@@ -570,13 +570,6 @@ public class HarFileSystem extends Filte
public String getName() {
return name;
}
-
-public ListString getChildren() {
- return children;
-}
-public String getFileName() {
- return name;
-}
public String getPartName() {
return partName;
}
@@ -647,15 +640,6 @@ public class HarFileSystem extends Filte
hstatus.getStartIndex(), hstatus.getLength(), bufferSize);
}
- /*
- * create throws an exception in Har filesystem.
- * The archive once created cannot be changed.
- */
- public FSDataOutputStream create(Path f, int bufferSize)
-throws IOException {
-throw new IOException(Har: Create not allowed);
- }
-
public FSDataOutputStream create(Path f,
FsPermission permission,
boolean overwrite,
@@ -1068,4 +1052,11 @@ public class HarFileSystem extends Filte
}
}
}
+
+ /*
+ * testing purposes only:
+ */
+ HarMetaData getMetadata() {
+return metadata;
+ }
}
svn commit: r1413776 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java src/test/java/org/apache/hadoop/fs/TestLocalDi
Author: bobby
Date: Mon Nov 26 18:47:20 2012
New Revision: 1413776
URL: http://svn.apache.org/viewvc?rev=1413776view=rev
Log:
HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
Veselovsky via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1413776r1=1413775r2=1413776view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov
26 18:47:20 2012
@@ -1155,6 +1155,9 @@ Release 0.23.6 - UNRELEASED
HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
Veselovsky via bobby)
+HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
+Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=1413776r1=1413775r2=1413776view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
Mon Nov 26 18:47:20 2012
@@ -481,12 +481,15 @@ public class LocalDirAllocator {
@Override
public Path next() {
-Path result = next;
+final Path result = next;
try {
advance();
} catch (IOException ie) {
throw new RuntimeException(Can't check existance of + next, ie);
}
+if (result == null) {
+ throw new NoSuchElementException();
+}
return result;
}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1413776r1=1413775r2=1413776view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Mon Nov 26 18:47:20 2012
@@ -22,6 +22,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
@@ -32,6 +34,7 @@ import org.junit.runners.Parameterized.P
import org.junit.Test;
import static org.junit.Assert.*;
+import static org.junit.Assume.*;
/** This test LocalDirAllocator works correctly;
* Every test case uses different buffer dirs to
@@ -318,7 +321,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testNoSideEffects() throws IOException {
-if (isWindows) return;
+assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -339,8 +342,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testGetLocalPathToRead() throws IOException {
-if (isWindows)
- return;
+assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -354,7 +356,60 @@ public class TestLocalDirAllocator {
Shell.execCommand(new String[] { chmod, u+w, BUFFER_DIR_ROOT });
rmBufferDirs();
}
+ }
+
+ /**
+ * Test that {@link LocalDirAllocator#getAllLocalPathsToRead(String,
Configuration)}
+ * returns correct filenames and file schema.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testGetAllLocalPathsToRead() throws IOException {
+assumeTrue(!isWindows);
+
+String dir0 = buildBufferDir(ROOT, 0);
+String dir1 = buildBufferDir(ROOT, 1);
+try {
+ conf.set(CONTEXT, dir0 + , + dir1);
+ assertTrue(localFs.mkdirs(new Path(dir0)));
+ assertTrue(localFs.mkdirs(new Path(dir1)));
+
+ localFs.create
svn commit: r1413778 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java src/test/java/org/apache/hadoop/fs
Author: bobby
Date: Mon Nov 26 18:48:24 2012
New Revision: 1413778
URL: http://svn.apache.org/viewvc?rev=1413778view=rev
Log:
svn merge -c 1413776 FIXES: HADOOP-9038. unit-tests for
AllocatorPerContext.PathIterator (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1413778r1=1413777r2=1413778view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Nov 26 18:48:24 2012
@@ -863,6 +863,9 @@ Release 0.23.6 - UNRELEASED
HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
Veselovsky via bobby)
+HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
+Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=1413778r1=1413777r2=1413778view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
Mon Nov 26 18:48:24 2012
@@ -478,12 +478,15 @@ public class LocalDirAllocator {
@Override
public Path next() {
-Path result = next;
+final Path result = next;
try {
advance();
} catch (IOException ie) {
throw new RuntimeException(Can't check existance of + next, ie);
}
+if (result == null) {
+ throw new NoSuchElementException();
+}
return result;
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1413778r1=1413777r2=1413778view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Mon Nov 26 18:48:24 2012
@@ -22,6 +22,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
@@ -32,6 +34,7 @@ import org.junit.runners.Parameterized.P
import org.junit.Test;
import static org.junit.Assert.*;
+import static org.junit.Assume.*;
/** This test LocalDirAllocator works correctly;
* Every test case uses different buffer dirs to
@@ -301,7 +304,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testNoSideEffects() throws IOException {
-if (isWindows) return;
+assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -322,8 +325,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testGetLocalPathToRead() throws IOException {
-if (isWindows)
- return;
+assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -337,7 +339,60 @@ public class TestLocalDirAllocator {
Shell.execCommand(new String[] { chmod, u+w, BUFFER_DIR_ROOT });
rmBufferDirs();
}
+ }
+
+ /**
+ * Test that {@link LocalDirAllocator#getAllLocalPathsToRead(String,
Configuration)}
+ * returns correct filenames and file schema.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testGetAllLocalPathsToRead() throws IOException {
+assumeTrue(!isWindows);
+
+String dir0 = buildBufferDir(ROOT, 0);
+String dir1
svn commit: r1413779 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java src/test/java/org/apache/hadoop
Author: bobby
Date: Mon Nov 26 18:51:18 2012
New Revision: 1413779
URL: http://svn.apache.org/viewvc?rev=1413779view=rev
Log:
svn merge -c 1413776 FIXES: HADOOP-9038. unit-tests for
AllocatorPerContext.PathIterator (Ivan A. Veselovsky via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1413779r1=1413778r2=1413779view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Nov 26 18:51:18 2012
@@ -20,6 +20,9 @@ Release 0.23.6 - UNRELEASED
HADOOP-8992. Enhance unit-test coverage of class HarFileSystem (Ivan A.
Veselovsky via bobby)
+HADOOP-9038. unit-tests for AllocatorPerContext.PathIterator (Ivan A.
+Veselovsky via bobby)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=1413779r1=1413778r2=1413779view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
Mon Nov 26 18:51:18 2012
@@ -478,12 +478,15 @@ public class LocalDirAllocator {
@Override
public Path next() {
-Path result = next;
+final Path result = next;
try {
advance();
} catch (IOException ie) {
throw new RuntimeException(Can't check existance of + next, ie);
}
+if (result == null) {
+ throw new NoSuchElementException();
+}
return result;
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1413779r1=1413778r2=1413779view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Mon Nov 26 18:51:18 2012
@@ -22,6 +22,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
@@ -32,6 +34,7 @@ import org.junit.runners.Parameterized.P
import org.junit.Test;
import static org.junit.Assert.*;
+import static org.junit.Assume.*;
/** This test LocalDirAllocator works correctly;
* Every test case uses different buffer dirs to
@@ -298,7 +301,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testNoSideEffects() throws IOException {
-if (isWindows) return;
+assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -319,8 +322,7 @@ public class TestLocalDirAllocator {
*/
@Test
public void testGetLocalPathToRead() throws IOException {
-if (isWindows)
- return;
+assumeTrue(!isWindows);
String dir = buildBufferDir(ROOT, 0);
try {
conf.set(CONTEXT, dir);
@@ -334,7 +336,60 @@ public class TestLocalDirAllocator {
Shell.execCommand(new String[] { chmod, u+w, BUFFER_DIR_ROOT });
rmBufferDirs();
}
+ }
+
+ /**
+ * Test that {@link LocalDirAllocator#getAllLocalPathsToRead(String,
Configuration)}
+ * returns correct filenames and file schema.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testGetAllLocalPathsToRead() throws IOException {
+assumeTrue(!isWindows);
+
+String dir0
svn commit: r1409817 - /hadoop/common/branches/branch-0.23.5/
Author: bobby Date: Thu Nov 15 14:54:29 2012 New Revision: 1409817 URL: http://svn.apache.org/viewvc?rev=1409817view=rev Log: Preparing for 0.23.5 release Added: hadoop/common/branches/branch-0.23.5/ (props changed) - copied from r1409816, hadoop/common/branches/branch-0.23/ Propchange: hadoop/common/branches/branch-0.23.5/ -- --- svn:ignore (added) +++ svn:ignore Thu Nov 15 14:54:29 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/branch-0.23.5/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Thu Nov 15 14:54:29 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1409818 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Thu Nov 15 14:57:23 2012 New Revision: 1409818 URL: http://svn.apache.org/viewvc?rev=1409818view=rev Log: Updated CHANGES.txt in preparation for 0.23.5 release Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1409818r1=1409817r2=1409818view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Nov 15 14:57:23 2012 @@ -1124,6 +1124,18 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via bobby) +Release 0.23.6 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -1159,7 +1171,7 @@ Release 0.23.5 - UNRELEASED HADOOP-9025. org.apache.hadoop.tools.TestCopyListing failing (Jonathan Eagles via jlowe) -Release 0.23.4 - UNRELEASED +Release 0.23.4 INCOMPATIBLE CHANGES
svn commit: r1409821 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Thu Nov 15 14:59:27 2012 New Revision: 1409821 URL: http://svn.apache.org/viewvc?rev=1409821view=rev Log: Updated CHANGES.txt in preparation for 0.23.5 release Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1409821r1=1409820r2=1409821view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Nov 15 14:59:27 2012 @@ -838,6 +838,18 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via bobby) +Release 0.23.6 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -873,7 +885,7 @@ Release 0.23.5 - UNRELEASED HADOOP-9025. org.apache.hadoop.tools.TestCopyListing failing (Jonathan Eagles via jlowe) -Release 0.23.4 - UNRELEASED +Release 0.23.4 INCOMPATIBLE CHANGES
svn commit: r1409823 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Thu Nov 15 15:03:27 2012 New Revision: 1409823 URL: http://svn.apache.org/viewvc?rev=1409823view=rev Log: Updated CHANGES.txt for 0.23.5 release Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1409823r1=1409822r2=1409823view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Thu Nov 15 15:03:27 2012 @@ -1,5 +1,17 @@ Hadoop Change Log +Release 0.23.6 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -61,7 +73,7 @@ Release 0.23.5 - UNRELEASED HADOOP-9025. org.apache.hadoop.tools.TestCopyListing failing (Jonathan Eagles via jlowe) -Release 0.23.4 - UNRELEASED +Release 0.23.4 INCOMPATIBLE CHANGES
svn commit: r1409824 - in /hadoop/common/branches/branch-0.23: ./ hadoop-assemblies/ hadoop-client/ hadoop-dist/ hadoop-minicluster/ hadoop-project-dist/ hadoop-project/ hadoop-tools/ hadoop-tools/had
Author: bobby Date: Thu Nov 15 15:06:49 2012 New Revision: 1409824 URL: http://svn.apache.org/viewvc?rev=1409824view=rev Log: Updated versions for branch-0.23 to be 0.23.6 Modified: hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml hadoop/common/branches/branch-0.23/hadoop-client/pom.xml hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml hadoop/common/branches/branch-0.23/hadoop-project-dist/pom.xml hadoop/common/branches/branch-0.23/hadoop-project/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-datajoin/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-pipes/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-rumen/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-tools-dist/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/pom.xml hadoop/common/branches/branch-0.23/pom.xml Modified: hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml?rev=1409824r1=1409823r2=1409824view=diff == --- hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml Thu Nov 15 15:06:49 2012 @@ -23,12 +23,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.5-SNAPSHOT/version +version0.23.6-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-assemblies/artifactId - version0.23.5-SNAPSHOT/version + version0.23.6-SNAPSHOT/version nameApache Hadoop Assemblies/name descriptionApache Hadoop Assemblies/description Modified: hadoop/common/branches/branch-0.23/hadoop-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-client/pom.xml?rev=1409824r1=1409823r2=1409824view=diff == --- hadoop/common/branches/branch-0.23/hadoop-client/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-client/pom.xml Thu Nov 15 15:06:49 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.5-SNAPSHOT/version +version0.23.6-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-client/artifactId - version0.23.5-SNAPSHOT/version + version0.23.6-SNAPSHOT/version packagingjar/packaging descriptionApache Hadoop Client/description Modified: hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml?rev=1409824r1=1409823r2=1409824view=diff == --- hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml Thu Nov 15 15:06:49 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.5-SNAPSHOT/version +version0.23.6-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-dist/artifactId - version0.23.5-SNAPSHOT/version + version0.23.6-SNAPSHOT/version descriptionApache Hadoop Distribution/description nameApache Hadoop Distribution/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml?rev=1409824r1=1409823r2=1409824view=diff == --- hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml Thu Nov 15 15:06:49 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.5-SNAPSHOT/version +version0.23.6-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-minicluster/artifactId - version0.23.5-SNAPSHOT/version + version0.23.6-SNAPSHOT/version packagingjar/packaging descriptionApache
svn commit: r1410018 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/se
Author: bobby
Date: Thu Nov 15 21:14:37 2012
New Revision: 1410018
URL: http://svn.apache.org/viewvc?rev=1410018view=rev
Log:
HADOOP-9035. Generalize setup of LoginContext (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1410018r1=1410017r2=1410018view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Nov
15 21:14:37 2012
@@ -369,6 +369,8 @@ Release 2.0.3-alpha - Unreleased
HADOO-8998. set Cache-Control no-cache header on all dynamic content.
(tucu)
+HADOOP-9035. Generalize setup of LoginContext (daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1410018r1=1410017r2=1410018view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Thu Nov 15 21:14:37 2012
@@ -294,14 +294,15 @@ public class Client {
}
}
+ AuthenticationMethod authentication;
if (token != null) {
-authMethod = AuthenticationMethod.TOKEN.getAuthMethod();
- } else if (UserGroupInformation.isSecurityEnabled()) {
-// eventually just use the ticket's authMethod
-authMethod = AuthMethod.KERBEROS;
- } else {
-authMethod = AuthMethod.SIMPLE;
+authentication = AuthenticationMethod.TOKEN;
+ } else if (ticket != null) {
+authentication = ticket.getRealAuthenticationMethod();
+ } else { // this only happens in lazy tests
+authentication = AuthenticationMethod.SIMPLE;
}
+ authMethod = authentication.getAuthMethod();
if (LOG.isDebugEnabled())
LOG.debug(Use + authMethod + authentication for protocol
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1410018r1=1410017r2=1410018view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Thu Nov 15 21:14:37 2012
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.security;
-import static
org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
import static
org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static
org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
@@ -133,7 +132,7 @@ public class UserGroupInformation {
}
Principal user = null;
// if we are using kerberos, try it out
- if (useKerberos) {
+ if (isAuthenticationMethodEnabled(AuthenticationMethod.KERBEROS)) {
user = getCanonicalUser(KerberosPrincipal.class);
if (LOG.isDebugEnabled()) {
LOG.debug(using kerberos user:+user);
@@ -191,8 +190,8 @@ public class UserGroupInformation {
static UgiMetrics metrics = UgiMetrics.create();
/** Are the static variables that depend on configuration initialized? */
private static boolean isInitialized = false;
- /** Should we use Kerberos configuration? */
- private static boolean useKerberos;
+ /** The auth method to use */
+ private static AuthenticationMethod authenticationMethod;
/** Server-side groups fetching service */
private static Groups groups;
/** Min time (in seconds) before relogin for Kerberos */
@@ -237,20 +236,7 @@ public class
svn commit: r1410024 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/security/ src/test/java/org/apac
Author: bobby
Date: Thu Nov 15 21:29:10 2012
New Revision: 1410024
URL: http://svn.apache.org/viewvc?rev=1410024view=rev
Log:
svn merge -c 1410018 FIXES: HADOOP-9035. Generalize setup of LoginContext
(daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1410024r1=1410023r2=1410024view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Nov 15 21:29:10 2012
@@ -81,6 +81,8 @@ Release 2.0.3-alpha - Unreleased
HADOO-8998. set Cache-Control no-cache header on all dynamic content.
(tucu)
+HADOOP-9035. Generalize setup of LoginContext (daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1410024r1=1410023r2=1410024view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Thu Nov 15 21:29:10 2012
@@ -292,14 +292,15 @@ public class Client {
}
}
+ AuthenticationMethod authentication;
if (token != null) {
-authMethod = AuthenticationMethod.TOKEN.getAuthMethod();
- } else if (UserGroupInformation.isSecurityEnabled()) {
-// eventually just use the ticket's authMethod
-authMethod = AuthMethod.KERBEROS;
- } else {
-authMethod = AuthMethod.SIMPLE;
+authentication = AuthenticationMethod.TOKEN;
+ } else if (ticket != null) {
+authentication = ticket.getRealAuthenticationMethod();
+ } else { // this only happens in lazy tests
+authentication = AuthenticationMethod.SIMPLE;
}
+ authMethod = authentication.getAuthMethod();
if (LOG.isDebugEnabled())
LOG.debug(Use + authMethod + authentication for protocol
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1410024r1=1410023r2=1410024view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Thu Nov 15 21:29:10 2012
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.security;
-import static
org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
@@ -131,7 +130,7 @@ public class UserGroupInformation {
}
Principal user = null;
// if we are using kerberos, try it out
- if (useKerberos) {
+ if (isAuthenticationMethodEnabled(AuthenticationMethod.KERBEROS)) {
user = getCanonicalUser(KerberosPrincipal.class);
if (LOG.isDebugEnabled()) {
LOG.debug(using kerberos user:+user);
@@ -189,8 +188,8 @@ public class UserGroupInformation {
static UgiMetrics metrics = UgiMetrics.create();
/** Are the static variables that depend on configuration initialized? */
private static boolean isInitialized = false;
- /** Should we use Kerberos configuration? */
- private static boolean useKerberos;
+ /** The auth method to use */
+ private static AuthenticationMethod authenticationMethod;
/** Server-side groups fetching service */
private static Groups groups
svn commit: r1408473 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/ip
Author: bobby
Date: Mon Nov 12 22:13:00 2012
New Revision: 1408473
URL: http://svn.apache.org/viewvc?rev=1408473view=rev
Log:
HADOOP-9021. Enforce configured SASL method on the server (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1408473r1=1408472r2=1408473view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov
12 22:13:00 2012
@@ -361,6 +361,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8860. Split MapReduce and YARN sections in documentation navigation.
(tomwhite via tucu)
+HADOOP-9021. Enforce configured SASL method on the server (daryn via
+bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1408473r1=1408472r2=1408473view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Mon Nov 12 22:13:00 2012
@@ -45,6 +45,7 @@ import java.security.PrivilegedException
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -87,7 +88,9 @@ import org.apache.hadoop.security.SaslRp
import org.apache.hadoop.security.SaslRpcServer.SaslDigestCallbackHandler;
import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
import org.apache.hadoop.security.SaslRpcServer.SaslStatus;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.PolicyProvider;
@@ -113,7 +116,7 @@ import com.google.common.annotations.Vis
@InterfaceStability.Evolving
public abstract class Server {
private final boolean authorize;
- private boolean isSecurityEnabled;
+ private EnumSetAuthMethod enabledAuthMethods;
private ExceptionsHandler exceptionsHandler = new ExceptionsHandler();
public void addTerseExceptions(Class?... exceptionClass) {
@@ -1334,34 +1337,9 @@ public abstract class Server {
if (authMethod == null) {
throw new IOException(Unable to read authentication method);
}
- boolean useSaslServer = isSecurityEnabled;
- final boolean clientUsingSasl;
- switch (authMethod) {
-case SIMPLE: { // no sasl for simple
- clientUsingSasl = false;
- break;
-}
-case DIGEST: { // always allow tokens if there's a secret manager
- useSaslServer |= (secretManager != null);
- clientUsingSasl = true;
- break;
-}
-default: {
- clientUsingSasl = true;
- break;
-}
- }
- if (useSaslServer) {
-saslServer = createSaslServer(authMethod);
- } else if (clientUsingSasl) { // security is off
-doSaslReply(SaslStatus.SUCCESS, new IntWritable(
-SaslRpcServer.SWITCH_TO_SIMPLE_AUTH), null, null);
-authMethod = AuthMethod.SIMPLE;
-// client has already sent the initial Sasl message and we
-// should ignore it. Both client and server should fall back
-// to simple auth from now on.
-skipInitialSaslHandshake = true;
- }
+
+ // this may create a SASL server, or switch us into SIMPLE
+ authMethod = initializeAuthContext(authMethod);
connectionHeaderBuf = null;
connectionHeaderRead = true;
@@ -1409,10 +1387,24 @@ public abstract class
svn commit: r1408488 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Mon Nov 12 22:43:07 2012 New Revision: 1408488 URL: http://svn.apache.org/viewvc?rev=1408488view=rev Log: HADOOP-9022. Hadoop distcp tool fails to copy file if -m 0 specified (Jonathan Eagles vai bobby) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1408488r1=1408487r2=1408488view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov 12 22:43:07 2012 @@ -1143,6 +1143,9 @@ Release 0.23.5 - UNRELEASED HADOOP-8986. Server$Call object is never released after it is sent (bobby) +HADOOP-9022. Hadoop distcp tool fails to copy file if -m 0 specified +(Jonathan Eagles vai bobby) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1408491 - in /hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src: main/java/org/apache/hadoop/tools/DistCpOptions.java test/java/org/apache/hadoop/tools/TestOptionsParser.java
Author: bobby
Date: Mon Nov 12 22:47:05 2012
New Revision: 1408491
URL: http://svn.apache.org/viewvc?rev=1408491view=rev
Log:
svn merge -c 1408488 FIXES: HADOOP-9022. Hadoop distcp tool fails to copy file
if -m 0 specified (Jonathan Eagles vai bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java?rev=1408491r1=1408490r2=1408491view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
Mon Nov 12 22:47:05 2012
@@ -274,7 +274,7 @@ public class DistCpOptions {
* @param maxMaps - Number of maps
*/
public void setMaxMaps(int maxMaps) {
-this.maxMaps = maxMaps;
+this.maxMaps = Math.max(maxMaps, 1);
}
/** Get the map bandwidth in MB
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1408491r1=1408490r2=1408491view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Mon Nov 12 22:47:05 2012
@@ -275,6 +275,13 @@ public class TestOptionsParser {
hdfs://localhost:8020/target/});
Assert.assertEquals(options.getMaxMaps(), 1);
+options = OptionsParser.parse(new String[] {
+-m,
+0,
+hdfs://localhost:8020/source/first,
+hdfs://localhost:8020/target/});
+Assert.assertEquals(options.getMaxMaps(), 1);
+
try {
OptionsParser.parse(new String[] {
-m,
svn commit: r1408491 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Mon Nov 12 22:47:05 2012 New Revision: 1408491 URL: http://svn.apache.org/viewvc?rev=1408491view=rev Log: svn merge -c 1408488 FIXES: HADOOP-9022. Hadoop distcp tool fails to copy file if -m 0 specified (Jonathan Eagles vai bobby) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1408491r1=1408490r2=1408491view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov 12 22:47:05 2012 @@ -860,6 +860,9 @@ Release 0.23.5 - UNRELEASED HADOOP-8986. Server$Call object is never released after it is sent (bobby) +HADOOP-9022. Hadoop distcp tool fails to copy file if -m 0 specified +(Jonathan Eagles vai bobby) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1408493 - in /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src: main/java/org/apache/hadoop/tools/DistCpOptions.java test/java/org/apache/hadoop/tools/TestOptionsParser.j
Author: bobby
Date: Mon Nov 12 22:52:01 2012
New Revision: 1408493
URL: http://svn.apache.org/viewvc?rev=1408493view=rev
Log:
svn merge -c 1408488 FIXES: HADOOP-9022. Hadoop distcp tool fails to copy file
if -m 0 specified (Jonathan Eagles vai bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java?rev=1408493r1=1408492r2=1408493view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
Mon Nov 12 22:52:01 2012
@@ -274,7 +274,7 @@ public class DistCpOptions {
* @param maxMaps - Number of maps
*/
public void setMaxMaps(int maxMaps) {
-this.maxMaps = maxMaps;
+this.maxMaps = Math.max(maxMaps, 1);
}
/** Get the map bandwidth in MB
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1408493r1=1408492r2=1408493view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Mon Nov 12 22:52:01 2012
@@ -275,6 +275,13 @@ public class TestOptionsParser {
hdfs://localhost:8020/target/});
Assert.assertEquals(options.getMaxMaps(), 1);
+options = OptionsParser.parse(new String[] {
+-m,
+0,
+hdfs://localhost:8020/source/first,
+hdfs://localhost:8020/target/});
+Assert.assertEquals(options.getMaxMaps(), 1);
+
try {
OptionsParser.parse(new String[] {
-m,
svn commit: r1407508 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java src/test/org/apache/hadoop/mapreduce/lib/input/TestNLineI
Author: bobby
Date: Fri Nov 9 15:56:05 2012
New Revision: 1407508
URL: http://svn.apache.org/viewvc?rev=1407508view=rev
Log:
MAPREDUCE-4782. NLineInputFormat skips first line of last InputSplit (Mark Fuhs
via bobby)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1407508r1=1407507r2=1407508view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Nov 9 15:56:05 2012
@@ -333,6 +333,9 @@ Release 1.1.1 - Unreleased
HDFS-4134. hadoop namenode and datanode entry points should return
negative exit code on bad arguments. (Steve Loughran via suresh)
+MAPREDUCE-4782. NLineInputFormat skips first line of last InputSplit
+(Mark Fuhs via bobby)
+
Release 1.1.0 - 2012.09.28
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java?rev=1407508r1=1407507r2=1407508view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
Fri Nov 9 15:56:05 2012
@@ -107,25 +107,14 @@ public class NLineInputFormat extends Fi
numLines++;
length += num;
if (numLines == numLinesPerSplit) {
- // NLineInputFormat uses LineRecordReader, which always reads
- // (and consumes) at least one character out of its upper split
- // boundary. So to make sure that each mapper gets N lines, we
- // move back the upper split limits of each split
- // by one character here.
- if (begin == 0) {
-splits.add(new FileSplit(fileName, begin, length - 1,
- new String[] {}));
- } else {
-splits.add(new FileSplit(fileName, begin - 1, length,
- new String[] {}));
- }
+ splits.add(createFileSplit(fileName, begin, length));
begin += length;
length = 0;
numLines = 0;
}
}
if (numLines != 0) {
-splits.add(new FileSplit(fileName, begin, length, new String[]{}));
+splits.add(createFileSplit(fileName, begin, length));
}
} finally {
if (lr != null) {
@@ -134,6 +123,23 @@ public class NLineInputFormat extends Fi
}
return splits;
}
+
+ /**
+ * NLineInputFormat uses LineRecordReader, which always reads
+ * (and consumes) at least one character out of its upper split
+ * boundary. So to make sure that each mapper gets N lines, we
+ * move back the upper split limits of each split
+ * by one character here.
+ * @param fileName Path of file
+ * @param begin the position of the first byte in the file to process
+ * @param length number of bytes in InputSplit
+ * @return FileSplit
+ */
+ protected static FileSplit createFileSplit(Path fileName, long begin, long
length) {
+return (begin == 0)
+? new FileSplit(fileName, begin, length - 1, new String[] {})
+: new FileSplit(fileName, begin - 1, length, new String[] {});
+ }
/**
* Set the number of lines per split
Modified:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java?rev=1407508r1=1407507r2=1407508view=diff
==
---
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java
(original)
+++
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java
Fri Nov 9 15:56:05 2012
@@ -49,37 +49,40 @@ public class TestNLineInputFormat extend
Job job = new Job(conf);
Path file = new Path(workDir, test.txt);
-int seed = new Random().nextInt();
-Random random = new Random(seed);
-
localFs.delete(workDir, true);
FileInputFormat.setInputPaths(job, workDir);
int numLinesPerMap = 5;
NLineInputFormat.setNumLinesPerSplit(job, numLinesPerMap);
-// for a variety of lengths
for (int length = 0; length MAX_LENGTH;
- length
svn commit: r1407622 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/ipc/
Author: bobby
Date: Fri Nov 9 20:40:09 2012
New Revision: 1407622
URL: http://svn.apache.org/viewvc?rev=1407622view=rev
Log:
HADOOP-9020. Add a SASL PLAIN server (daryn via bobby)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1407622r1=1407621r2=1407622view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Nov
9 20:40:09 2012
@@ -291,6 +291,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8597. Permit FsShell's text command to read Avro files.
(Ivan Vladimirov Ivanov via cutting)
+HADOOP-9020. Add a SASL PLAIN server (daryn via bobby)
+
IMPROVEMENTS
HADOOP-8789. Tests setLevel(Level.OFF) should be Level.ERROR.
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java?rev=1407622view=auto
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java
Fri Nov 9 20:40:09 2012
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.security.Provider;
+import java.util.Map;
+
+import javax.security.auth.callback.*;
+import javax.security.sasl.AuthorizeCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
+import javax.security.sasl.SaslServerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SaslPlainServer implements SaslServer {
+ @SuppressWarnings(serial)
+ public static class SecurityProvider extends Provider {
+public SecurityProvider() {
+ super(SaslPlainServer, 1.0, SASL PLAIN Authentication Server);
+ put(SaslServerFactory.PLAIN,
+ SaslPlainServerFactory.class.getName());
+}
+ }
+
+ public static class SaslPlainServerFactory implements SaslServerFactory {
+@Override
+public SaslServer createSaslServer(String mechanism, String protocol,
+String serverName, MapString,? props, CallbackHandler cbh)
+throws SaslException {
+ return PLAIN.equals(mechanism) ? new SaslPlainServer(cbh) : null;
+}
+@Override
+public String[] getMechanismNames(MapString,? props){
+ return (props == null) ||
false.equals(props.get(Sasl.POLICY_NOPLAINTEXT))
+ ? new String[]{PLAIN}
+ : new String[0];
+}
+ }
+
+ private CallbackHandler cbh;
+ private boolean completed;
+ private String authz;
+
+ SaslPlainServer(CallbackHandler callback) {
+this.cbh = callback;
+ }
+
+ @Override
+ public String getMechanismName() {
+return PLAIN;
+ }
+
+ @Override
+ public byte[] evaluateResponse(byte[] response) throws SaslException {
+if (completed) {
+ throw new IllegalStateException(PLAIN authentication has completed);
+}
+if (response == null) {
+ throw new IllegalArgumentException(Received null response);
+}
+try {
+ String payload;
+ try
svn commit: r1407623 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/ipc/
Author: bobby
Date: Fri Nov 9 20:45:09 2012
New Revision: 1407623
URL: http://svn.apache.org/viewvc?rev=1407623view=rev
Log:
svn emrge -c 1407622 FIXES: HADOOP-9020. Add a SASL PLAIN server (daryn via
bobby)
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java
- copied unchanged from r1407622,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPlainServer.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1407623r1=1407622r2=1407623view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Nov 9 20:45:09 2012
@@ -9,6 +9,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8597. Permit FsShell's text command to read Avro files.
(Ivan Vladimirov Ivanov via cutting)
+HADOOP-9020. Add a SASL PLAIN server (daryn via bobby)
+
IMPROVEMENTS
HADOOP-8789. Tests setLevel(Level.OFF) should be Level.ERROR.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java?rev=1407623r1=1407622r2=1407623view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
Fri Nov 9 20:45:09 2012
@@ -23,6 +23,7 @@ import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
+import java.security.Security;
import java.util.Map;
import java.util.TreeMap;
@@ -89,6 +90,7 @@ public class SaslRpcServer {
SASL_PROPS.put(Sasl.QOP, saslQOP.getSaslQop());
SASL_PROPS.put(Sasl.SERVER_AUTH, true);
+Security.addProvider(new SaslPlainServer.SecurityProvider());
}
static String encodeIdentifier(byte[] identifier) {
@@ -138,7 +140,8 @@ public class SaslRpcServer {
public static enum AuthMethod {
SIMPLE((byte) 80, ),
KERBEROS((byte) 81, GSSAPI),
-DIGEST((byte) 82, DIGEST-MD5);
+DIGEST((byte) 82, DIGEST-MD5),
+PLAIN((byte) 83, PLAIN);
/** The code for this method. */
public final byte code;
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java?rev=1407623r1=1407622r2=1407623view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Fri Nov 9 20:45:09 2012
@@ -27,12 +27,13 @@ import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
+import java.security.Security;
import java.util.Collection;
import java.util.Set;
import java.util.regex.Pattern;
-import javax.security.sasl.Sasl;
-
+import javax.security.auth.callback.*;
+import javax.security.sasl.*;
import junit.framework.Assert;
import org.apache.commons.logging.Log;
@@ -44,6 +45,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.*;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
@@ -53,7 +55,6 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import
svn commit: r1406684 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/security/UserGroupInformation.java src/test/java/org/apache/hadoop/ipc/T
Author: bobby
Date: Wed Nov 7 15:56:46 2012
New Revision: 1406684
URL: http://svn.apache.org/viewvc?rev=1406684view=rev
Log:
HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn via
bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1406684r1=1406683r2=1406684view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov
7 15:56:46 2012
@@ -342,6 +342,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9010. Map UGI authenticationMethod to RPC authMethod (daryn via
bobby)
+HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn
+via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1406684r1=1406683r2=1406684view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
Wed Nov 7 15:56:46 2012
@@ -238,14 +238,17 @@ public class UserGroupInformation {
*/
private static synchronized void initUGI(Configuration conf) {
AuthenticationMethod auth = SecurityUtil.getAuthenticationMethod(conf);
-if (auth == AuthenticationMethod.SIMPLE) {
- useKerberos = false;
-} else if (auth == AuthenticationMethod.KERBEROS) {
- useKerberos = true;
-} else {
- throw new IllegalArgumentException(Invalid attribute value for +
- HADOOP_SECURITY_AUTHENTICATION +
- of + auth);
+switch (auth) {
+ case SIMPLE:
+useKerberos = false;
+break;
+ case KERBEROS:
+useKerberos = true;
+break;
+ default:
+throw new IllegalArgumentException(Invalid attribute value for +
+ HADOOP_SECURITY_AUTHENTICATION +
+of + auth);
}
try {
kerberosMinSecondsBeforeRelogin = 1000L * conf.getLong(
@@ -637,19 +640,20 @@ public class UserGroupInformation {
try {
Subject subject = new Subject();
LoginContext login;
+AuthenticationMethod authenticationMethod;
if (isSecurityEnabled()) {
+ authenticationMethod = AuthenticationMethod.KERBEROS;
login =
newLoginContext(HadoopConfiguration.USER_KERBEROS_CONFIG_NAME,
subject, new HadoopConfiguration());
} else {
+ authenticationMethod = AuthenticationMethod.SIMPLE;
login = newLoginContext(HadoopConfiguration.SIMPLE_CONFIG_NAME,
subject, new HadoopConfiguration());
}
login.login();
loginUser = new UserGroupInformation(subject);
loginUser.setLogin(login);
-loginUser.setAuthenticationMethod(isSecurityEnabled() ?
- AuthenticationMethod.KERBEROS :
- AuthenticationMethod.SIMPLE);
+loginUser.setAuthenticationMethod(authenticationMethod);
loginUser = new UserGroupInformation(login.getSubject());
String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
if (fileLocation != null) {
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java?rev=1406684r1=1406683r2=1406684view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Wed Nov 7 15:56:46 2012
svn commit: r1406689 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/security/SaslRpcClient.java
Author: bobby
Date: Wed Nov 7 16:05:03 2012
New Revision: 1406689
URL: http://svn.apache.org/viewvc?rev=1406689view=rev
Log:
HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1406689r1=1406688r2=1406689view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov
7 16:05:03 2012
@@ -345,6 +345,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn
via bobby)
+HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java?rev=1406689r1=1406688r2=1406689view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
Wed Nov 7 16:05:03 2012
@@ -25,6 +25,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Map;
import javax.security.auth.callback.Callback;
import javax.security.auth.callback.CallbackHandler;
@@ -45,6 +46,7 @@ import org.apache.hadoop.io.WritableUtil
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SaslRpcServer.SaslStatus;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -69,40 +71,48 @@ public class SaslRpcClient {
public SaslRpcClient(AuthMethod method,
Token? extends TokenIdentifier token, String serverPrincipal)
throws IOException {
+String saslUser = null;
+String saslProtocol = null;
+String saslServerName = null;
+MapString, String saslProperties = SaslRpcServer.SASL_PROPS;
+CallbackHandler saslCallback = null;
+
switch (method) {
-case DIGEST:
- if (LOG.isDebugEnabled())
-LOG.debug(Creating SASL + AuthMethod.DIGEST.getMechanismName()
-+ client to authenticate to service at + token.getService());
- saslClient = Sasl.createSaslClient(new String[] { AuthMethod.DIGEST
- .getMechanismName() }, null, null, SaslRpcServer.SASL_DEFAULT_REALM,
- SaslRpcServer.SASL_PROPS, new SaslClientCallbackHandler(token));
- break;
-case KERBEROS:
- if (LOG.isDebugEnabled()) {
-LOG.debug(Creating SASL + AuthMethod.KERBEROS.getMechanismName()
-+ client. Server's Kerberos principal name is
-+ serverPrincipal);
- }
- if (serverPrincipal == null || serverPrincipal.length() == 0) {
-throw new IOException(
-Failed to specify server's Kerberos principal name);
- }
- String names[] = SaslRpcServer.splitKerberosName(serverPrincipal);
- if (names.length != 3) {
-throw new IOException(
- Kerberos principal name does NOT have the expected hostname part:
-+ serverPrincipal);
- }
- saslClient = Sasl.createSaslClient(new String[] { AuthMethod.KERBEROS
- .getMechanismName() }, null, names[0], names[1],
- SaslRpcServer.SASL_PROPS, null);
- break;
-default:
- throw new IOException(Unknown authentication method + method);
+ case DIGEST: {
+saslServerName = SaslRpcServer.SASL_DEFAULT_REALM;
+saslCallback = new SaslClientCallbackHandler(token);
+break;
+ }
+ case KERBEROS: {
+if (serverPrincipal == null || serverPrincipal.isEmpty()) {
+ throw new IOException(
+ Failed to specify server's Kerberos principal name);
+}
+KerberosName name = new KerberosName(serverPrincipal);
+saslProtocol = name.getServiceName();
+saslServerName = name.getHostName();
+if (saslServerName == null) {
+ throw new IOException
svn commit: r1406692 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/security/SaslRpcClient.java
Author: bobby
Date: Wed Nov 7 16:08:24 2012
New Revision: 1406692
URL: http://svn.apache.org/viewvc?rev=1406692view=rev
Log:
svn merge -c 1406689 FIXES: HADOOP-9014. Standardize creation of SaslRpcClients
(daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1406692r1=1406691r2=1406692view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Nov 7 16:08:24 2012
@@ -70,6 +70,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9013. UGI should not hardcode loginUser's authenticationType (daryn
via bobby)
+HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java?rev=1406692r1=1406691r2=1406692view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
Wed Nov 7 16:08:24 2012
@@ -25,6 +25,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Map;
import javax.security.auth.callback.Callback;
import javax.security.auth.callback.CallbackHandler;
@@ -45,6 +46,7 @@ import org.apache.hadoop.io.WritableUtil
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SaslRpcServer.SaslStatus;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -69,40 +71,48 @@ public class SaslRpcClient {
public SaslRpcClient(AuthMethod method,
Token? extends TokenIdentifier token, String serverPrincipal)
throws IOException {
+String saslUser = null;
+String saslProtocol = null;
+String saslServerName = null;
+MapString, String saslProperties = SaslRpcServer.SASL_PROPS;
+CallbackHandler saslCallback = null;
+
switch (method) {
-case DIGEST:
- if (LOG.isDebugEnabled())
-LOG.debug(Creating SASL + AuthMethod.DIGEST.getMechanismName()
-+ client to authenticate to service at + token.getService());
- saslClient = Sasl.createSaslClient(new String[] { AuthMethod.DIGEST
- .getMechanismName() }, null, null, SaslRpcServer.SASL_DEFAULT_REALM,
- SaslRpcServer.SASL_PROPS, new SaslClientCallbackHandler(token));
- break;
-case KERBEROS:
- if (LOG.isDebugEnabled()) {
-LOG.debug(Creating SASL + AuthMethod.KERBEROS.getMechanismName()
-+ client. Server's Kerberos principal name is
-+ serverPrincipal);
- }
- if (serverPrincipal == null || serverPrincipal.length() == 0) {
-throw new IOException(
-Failed to specify server's Kerberos principal name);
- }
- String names[] = SaslRpcServer.splitKerberosName(serverPrincipal);
- if (names.length != 3) {
-throw new IOException(
- Kerberos principal name does NOT have the expected hostname part:
-+ serverPrincipal);
- }
- saslClient = Sasl.createSaslClient(new String[] { AuthMethod.KERBEROS
- .getMechanismName() }, null, names[0], names[1],
- SaslRpcServer.SASL_PROPS, null);
- break;
-default:
- throw new IOException(Unknown authentication method + method);
+ case DIGEST: {
+saslServerName = SaslRpcServer.SASL_DEFAULT_REALM;
+saslCallback = new SaslClientCallbackHandler(token);
+break;
+ }
+ case KERBEROS: {
+if (serverPrincipal == null || serverPrincipal.isEmpty()) {
+ throw new IOException(
+ Failed to specify server's Kerberos principal name);
+}
+KerberosName name = new KerberosName(serverPrincipal);
+saslProtocol
svn commit: r1406851 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Author: bobby
Date: Wed Nov 7 22:50:48 2012
New Revision: 1406851
URL: http://svn.apache.org/viewvc?rev=1406851view=rev
Log:
HADOOP-9015. Standardize creation of SaslRpcServers (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1406851r1=1406850r2=1406851view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov
7 22:50:48 2012
@@ -347,6 +347,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby)
+HADOOP-9015. Standardize creation of SaslRpcServers (daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1406851r1=1406850r2=1406851view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Wed Nov 7 22:50:48 2012
@@ -57,6 +57,7 @@ import java.util.concurrent.BlockingQueu
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
+import javax.security.auth.callback.CallbackHandler;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import javax.security.sasl.SaslServer;
@@ -87,6 +88,7 @@ import org.apache.hadoop.security.SaslRp
import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
import org.apache.hadoop.security.SaslRpcServer.SaslStatus;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
@@ -1078,7 +1080,6 @@ public abstract class Server {
IpcConnectionContextProto connectionContext;
String protocolName;
-boolean useSasl;
SaslServer saslServer;
private AuthMethod authMethod;
private boolean saslContextEstablished;
@@ -1194,49 +1195,6 @@ public abstract class Server {
if (!saslContextEstablished) {
byte[] replyToken = null;
try {
- if (saslServer == null) {
-switch (authMethod) {
-case DIGEST:
- if (secretManager == null) {
-throw new AccessControlException(
-Server is not configured to do DIGEST authentication.);
- }
- secretManager.checkAvailableForRead();
- saslServer = Sasl.createSaslServer(AuthMethod.DIGEST
- .getMechanismName(), null, SaslRpcServer.SASL_DEFAULT_REALM,
- SaslRpcServer.SASL_PROPS, new SaslDigestCallbackHandler(
- secretManager, this));
- break;
-default:
- UserGroupInformation current = UserGroupInformation
- .getCurrentUser();
- String fullName = current.getUserName();
- if (LOG.isDebugEnabled())
-LOG.debug(Kerberos principal name is + fullName);
- final String names[] = SaslRpcServer.splitKerberosName(fullName);
- if (names.length != 3) {
-throw new AccessControlException(
-Kerberos principal name does NOT have the expected
-+ hostname part: + fullName);
- }
- current.doAs(new PrivilegedExceptionActionObject() {
-@Override
-public Object run() throws SaslException {
- saslServer = Sasl.createSaslServer(AuthMethod.KERBEROS
- .getMechanismName(), names[0], names[1],
- SaslRpcServer.SASL_PROPS, new SaslGssCallbackHandler());
- return null;
-}
- });
-}
-if (saslServer == null)
- throw new AccessControlException(
- Unable to find SASL
svn commit: r1406853 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java src/test/java/org/apache/hadoop/ipc/TestSasl
Author: bobby
Date: Wed Nov 7 22:55:02 2012
New Revision: 1406853
URL: http://svn.apache.org/viewvc?rev=1406853view=rev
Log:
svn merge -c 1406851 FIXES: HADOOP-9015. Standardize creation of SaslRpcServers
(daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1406853r1=1406852r2=1406853view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Nov 7 22:55:02 2012
@@ -72,6 +72,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9014. Standardize creation of SaslRpcClients (daryn via bobby)
+HADOOP-9015. Standardize creation of SaslRpcServers (daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1406853r1=1406852r2=1406853view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Wed Nov 7 22:55:02 2012
@@ -55,6 +55,7 @@ import java.util.concurrent.BlockingQueu
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
+import javax.security.auth.callback.CallbackHandler;
import javax.security.sasl.Sasl;
import javax.security.sasl.SaslException;
import javax.security.sasl.SaslServer;
@@ -84,6 +85,7 @@ import org.apache.hadoop.security.SaslRp
import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
import org.apache.hadoop.security.SaslRpcServer.SaslStatus;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
@@ -1037,7 +1039,6 @@ public abstract class Server {
IpcConnectionContextProto connectionContext;
String protocolName;
-boolean useSasl;
SaslServer saslServer;
private AuthMethod authMethod;
private boolean saslContextEstablished;
@@ -1153,49 +1154,6 @@ public abstract class Server {
if (!saslContextEstablished) {
byte[] replyToken = null;
try {
- if (saslServer == null) {
-switch (authMethod) {
-case DIGEST:
- if (secretManager == null) {
-throw new AccessControlException(
-Server is not configured to do DIGEST authentication.);
- }
- secretManager.checkAvailableForRead();
- saslServer = Sasl.createSaslServer(AuthMethod.DIGEST
- .getMechanismName(), null, SaslRpcServer.SASL_DEFAULT_REALM,
- SaslRpcServer.SASL_PROPS, new SaslDigestCallbackHandler(
- secretManager, this));
- break;
-default:
- UserGroupInformation current = UserGroupInformation
- .getCurrentUser();
- String fullName = current.getUserName();
- if (LOG.isDebugEnabled())
-LOG.debug(Kerberos principal name is + fullName);
- final String names[] = SaslRpcServer.splitKerberosName(fullName);
- if (names.length != 3) {
-throw new AccessControlException(
-Kerberos principal name does NOT have the expected
-+ hostname part: + fullName);
- }
- current.doAs(new PrivilegedExceptionActionObject() {
-@Override
-public Object run() throws SaslException {
- saslServer = Sasl.createSaslServer(AuthMethod.KERBEROS
- .getMechanismName(), names[0], names[1],
- SaslRpcServer.SASL_PROPS, new SaslGssCallbackHandler());
- return null
svn commit: r1405904 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/ipc/ src/test/java/org/apache/hadoop/se
Author: bobby
Date: Mon Nov 5 18:26:49 2012
New Revision: 1405904
URL: http://svn.apache.org/viewvc?rev=1405904view=rev
Log:
HADOOP-9009. Add SecurityUtil methods to get/set authentication method (daryn
via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithSecurityOn.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1405904r1=1405903r2=1405904view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov
5 18:26:49 2012
@@ -333,6 +333,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8985. Add namespace declarations in .proto files for languages
other than java. (Binglin Chan via suresh)
+HADOOP-9009. Add SecurityUtil methods to get/set authentication method
+(daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1405904r1=1405903r2=1405904view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
Mon Nov 5 18:26:49 2012
@@ -16,6 +16,8 @@
*/
package org.apache.hadoop.security;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -44,6 +46,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.ssl.SSLFactory;
@@ -665,4 +668,22 @@ public class SecurityUtil {
}
}
+ public static AuthenticationMethod getAuthenticationMethod(Configuration
conf) {
+String value = conf.get(HADOOP_SECURITY_AUTHENTICATION, simple);
+try {
+ return Enum.valueOf(AuthenticationMethod.class, value.toUpperCase());
+} catch (IllegalArgumentException iae) {
+ throw new IllegalArgumentException(Invalid attribute value for +
+ HADOOP_SECURITY_AUTHENTICATION + of + value);
+}
+ }
+
+ public static void setAuthenticationMethod(
+ AuthenticationMethod authenticationMethod, Configuration conf) {
+if (authenticationMethod == null) {
+ authenticationMethod = AuthenticationMethod.SIMPLE;
+}
+conf.set(HADOOP_SECURITY_AUTHENTICATION,
+ authenticationMethod.toString().toLowerCase());
+ }
}
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1405904r1=1405903r2=1405904view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main
svn commit: r1405905 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/ipc/ src/test/java/org/apac
Author: bobby
Date: Mon Nov 5 18:32:33 2012
New Revision: 1405905
URL: http://svn.apache.org/viewvc?rev=1405905view=rev
Log:
svn merge -c 1405904 FIXES: HADOOP-9009. Add SecurityUtil methods to get/set
authentication method (daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/MiniRPCBenchmark.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1405905r1=1405904r2=1405905view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Nov 5 18:32:33 2012
@@ -61,6 +61,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8985. Add namespace declarations in .proto files for languages
other than java. (Binglin Chan via suresh)
+HADOOP-9009. Add SecurityUtil methods to get/set authentication method
+(daryn via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1405905r1=1405904r2=1405905view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
Mon Nov 5 18:32:33 2012
@@ -16,6 +16,8 @@
*/
package org.apache.hadoop.security;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@@ -44,6 +46,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.ssl.SSLFactory;
@@ -665,4 +668,22 @@ public class SecurityUtil {
}
}
+ public static AuthenticationMethod getAuthenticationMethod(Configuration
conf) {
+String value = conf.get(HADOOP_SECURITY_AUTHENTICATION, simple);
+try {
+ return Enum.valueOf(AuthenticationMethod.class, value.toUpperCase());
+} catch (IllegalArgumentException iae) {
+ throw new IllegalArgumentException(Invalid attribute value for +
+ HADOOP_SECURITY_AUTHENTICATION + of + value);
+}
+ }
+
+ public static void setAuthenticationMethod(
+ AuthenticationMethod authenticationMethod, Configuration conf) {
+if (authenticationMethod == null) {
+ authenticationMethod = AuthenticationMethod.SIMPLE;
+}
+conf.set(HADOOP_SECURITY_AUTHENTICATION,
+ authenticationMethod.toString().toLowerCase());
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java?rev=1405905r1=1405904r2=1405905view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org
svn commit: r1405910 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/security/ src/test/java/org/apache/hadoop/se
Author: bobby
Date: Mon Nov 5 18:37:39 2012
New Revision: 1405910
URL: http://svn.apache.org/viewvc?rev=1405910view=rev
Log:
HADOOP-9010. Map UGI authenticationMethod to RPC authMethod (daryn via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1405910r1=1405909r2=1405910view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon Nov
5 18:37:39 2012
@@ -336,6 +336,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9009. Add SecurityUtil methods to get/set authentication method
(daryn via bobby)
+HADOOP-9010. Map UGI authenticationMethod to RPC authMethod (daryn via
+bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1405910r1=1405909r2=1405910view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Mon Nov 5 18:37:39 2012
@@ -69,6 +69,7 @@ import org.apache.hadoop.security.SaslRp
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenInfo;
@@ -295,8 +296,9 @@ public class Client {
}
if (token != null) {
-authMethod = AuthMethod.DIGEST;
+authMethod = AuthenticationMethod.TOKEN.getAuthMethod();
} else if (UserGroupInformation.isSecurityEnabled()) {
+// eventually just use the ticket's authMethod
authMethod = AuthMethod.KERBEROS;
} else {
authMethod = AuthMethod.SIMPLE;
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1405910r1=1405909r2=1405910view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Mon Nov 5 18:37:39 2012
@@ -1526,11 +1526,11 @@ public abstract class Server {
if (!useSasl) {
user = protocolUser;
if (user != null) {
- user.setAuthenticationMethod(AuthMethod.SIMPLE.authenticationMethod);
+ user.setAuthenticationMethod(AuthMethod.SIMPLE);
}
} else {
// user is authenticated
-user.setAuthenticationMethod(authMethod.authenticationMethod);
+user.setAuthenticationMethod(authMethod);
//Now we check if this is a proxy user case. If the protocol user is
//different from the 'user', it is a proxy user scenario. However,
//this is not allowed if user authenticated with DIGEST.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java?rev=1405910r1=1405909r2=1405910view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src
svn commit: r1405911 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/ipc/ src/main/java/org/apache/hadoop/security/ src/test/java/org/apac
Author: bobby
Date: Mon Nov 5 18:40:39 2012
New Revision: 1405911
URL: http://svn.apache.org/viewvc?rev=1405911view=rev
Log:
svn merge -c 1405910 FIXES: HADOOP-9010. Map UGI authenticationMethod to RPC
authMethod (daryn via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1405911r1=1405910r2=1405911view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Nov 5 18:40:39 2012
@@ -64,6 +64,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-9009. Add SecurityUtil methods to get/set authentication method
(daryn via bobby)
+HADOOP-9010. Map UGI authenticationMethod to RPC authMethod (daryn via
+bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java?rev=1405911r1=1405910r2=1405911view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
Mon Nov 5 18:40:39 2012
@@ -69,6 +69,7 @@ import org.apache.hadoop.security.SaslRp
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenInfo;
@@ -293,8 +294,9 @@ public class Client {
}
if (token != null) {
-authMethod = AuthMethod.DIGEST;
+authMethod = AuthenticationMethod.TOKEN.getAuthMethod();
} else if (UserGroupInformation.isSecurityEnabled()) {
+// eventually just use the ticket's authMethod
authMethod = AuthMethod.KERBEROS;
} else {
authMethod = AuthMethod.SIMPLE;
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1405911r1=1405910r2=1405911view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Mon Nov 5 18:40:39 2012
@@ -1485,11 +1485,11 @@ public abstract class Server {
if (!useSasl) {
user = protocolUser;
if (user != null) {
- user.setAuthenticationMethod(AuthMethod.SIMPLE.authenticationMethod);
+ user.setAuthenticationMethod(AuthMethod.SIMPLE);
}
} else {
// user is authenticated
-user.setAuthenticationMethod(authMethod.authenticationMethod);
+user.setAuthenticationMethod(authMethod);
//Now we check if this is a proxy user case. If the protocol user is
//different from the 'user', it is a proxy user scenario. However,
//this is not allowed if user authenticated with DIGEST.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop
svn commit: r1403745 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java
Author: bobby
Date: Tue Oct 30 15:46:15 2012
New Revision: 1403745
URL: http://svn.apache.org/viewvc?rev=1403745view=rev
Log:
HADOOP-8986. Server$Call object is never released after it is sent (bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1403745r1=1403744r2=1403745view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct
30 15:46:15 2012
@@ -1106,6 +1106,8 @@ Release 0.23.5 - UNRELEASED
HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename
contains a colon (jlowe via bobby)
+HADOOP-8986. Server$Call object is never released after it is sent (bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1403745r1=1403744r2=1403745view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Tue Oct 30 15:46:15 2012
@@ -974,6 +974,8 @@ public abstract class Server {
return true;
}
if (!call.rpcResponse.hasRemaining()) {
+//Clear out the response buffer so it can be collected
+call.rpcResponse = null;
call.connection.decRpcCount();
if (numElements == 1) {// last call fully processes.
done = true; // no more data for this channel.
svn commit: r1403747 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java
Author: bobby
Date: Tue Oct 30 15:51:29 2012
New Revision: 1403747
URL: http://svn.apache.org/viewvc?rev=1403747view=rev
Log:
svn merge -c 1403745 FIXES: HADOOP-8986. Server$Call object is never released
after it is sent (bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1403747r1=1403746r2=1403747view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 30 15:51:29 2012
@@ -836,6 +836,8 @@ Release 0.23.5 - UNRELEASED
HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename
contains a colon (jlowe via bobby)
+HADOOP-8986. Server$Call object is never released after it is sent (bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1403747r1=1403746r2=1403747view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Tue Oct 30 15:51:29 2012
@@ -933,6 +933,8 @@ public abstract class Server {
return true;
}
if (!call.rpcResponse.hasRemaining()) {
+//Clear out the response buffer so it can be collected
+call.rpcResponse = null;
call.connection.decRpcCount();
if (numElements == 1) {// last call fully processes.
done = true; // no more data for this channel.
svn commit: r1403750 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/ipc/Server.java
Author: bobby
Date: Tue Oct 30 15:55:48 2012
New Revision: 1403750
URL: http://svn.apache.org/viewvc?rev=1403750view=rev
Log:
svn merge -c 1403745 FIXES: HADOOP-8986. Server$Call object is never released
after it is sent (bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1403750r1=1403749r2=1403750view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 30 15:55:48 2012
@@ -53,6 +53,8 @@ Release 0.23.5 - UNRELEASED
HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename
contains a colon (jlowe via bobby)
+HADOOP-8986. Server$Call object is never released after it is sent (bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1403750r1=1403749r2=1403750view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Tue Oct 30 15:55:48 2012
@@ -827,6 +827,8 @@ public abstract class Server {
return true;
}
if (!call.response.hasRemaining()) {
+//Clear out the response buffer so it can be collected
+call.response = null;
call.connection.decRpcCount();
if (numElements == 1) {// last call fully processes.
done = true; // no more data for this channel.
svn commit: r1401803 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/util/ src/test/java/org/apache/hadoop/util/
Author: bobby
Date: Wed Oct 24 18:12:57 2012
New Revision: 1401803
URL: http://svn.apache.org/viewvc?rev=1401803view=rev
Log:
svn merge -c 1399005 FIXES: HADOOP-8926. hadoop.util.PureJavaCrc32 cache
hit-ratio is low for static data (Gopal V via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1401803r1=1401802r2=1401803view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Oct 24 18:12:57 2012
@@ -30,6 +30,9 @@ Release 0.23.5 - UNRELEASED
HADOOP-8819. Incorrectly is used instead of in some file system
implementations. (Brandon Li via suresh)
+HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
+data (Gopal V via bobby)
+
BUG FIXES
HADOOP-8386. hadoop script doesn't work if 'cd' prints to stdout
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java?rev=1401803r1=1401802r2=1401803view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
Wed Oct 24 18:12:57 2012
@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Ch
/** {@inheritDoc} */
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc = 8);
- int c2 = b[off++] ^ (localCrc = 8);
- int c3 = b[off++] ^ (localCrc = 8);
- localCrc = (T8_7[c0 0xff] ^ T8_6[c1 0xff])
- ^ (T8_5[c2 0xff] ^ T8_4[c3 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) 0xff;
+ final int c1 =(b[off+1] ^ (localCrc = 8)) 0xff;
+ final int c2 =(b[off+2] ^ (localCrc = 8)) 0xff;
+ final int c3 =(b[off+3] ^ (localCrc = 8)) 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] 0xff;
+ final int c5 = b[off+5] 0xff;
+ final int c6 = b[off+6] 0xff;
+ final int c7 = b[off+7] 0xff;
- localCrc ^= (T8_3[b[off++] 0xff] ^ T8_2[b[off++] 0xff])
- ^ (T8_1[b[off++] 0xff] ^ T8_0[b[off++] 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
-while(len 0) {
- localCrc = (localCrc 8) ^ T8_0[(localCrc ^ b[off++]) 0xff];
- len--;
+
+/* loop unroll - duff's device style */
+switch(len) {
+ case 7: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 6: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 5: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 4: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 3: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 2: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 1: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ default:
+/* nothing */
}
// Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Ch
/** {@inheritDoc} */
final public void update(int b) {
-crc = (crc 8) ^ T8_0[(crc ^ b) 0xff];
+crc = (crc 8) ^ T[T8_0_start + ((crc ^ b) 0xff)];
}
/*
* CRC-32 lookup tables generated by the polynomial 0xEDB88320.
* See also TestPureJavaCrc32.Table.
*/
- private static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256
svn commit: r1401321 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/ src/main/java/org/apache/hadoop/fs/ src/main/native/src/org/apache/hadoop/io/nativeio/ src/main/native/src/o
Author: bobby
Date: Tue Oct 23 15:32:25 2012
New Revision: 1401321
URL: http://svn.apache.org/viewvc?rev=1401321view=rev
Log:
HADOOP-8811. Compile hadoop native library in FreeBSD (Radim Kolar via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/getGroup.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1401321r1=1401320r2=1401321view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct
23 15:32:25 2012
@@ -1094,6 +1094,9 @@ Release 0.23.5 - UNRELEASED
HADOOP-8906. paths with multiple globs are unreliable. (Daryn Sharp via
jlowe)
+HADOOP-8811. Compile hadoop native library in FreeBSD (Radim Kolar via
+bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1401321r1=1401320r2=1401321view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/CMakeLists.txt
Tue Oct 23 15:32:25 2012
@@ -67,6 +67,9 @@ macro(set_find_shared_library_version LV
IF(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
# Mac OS uses .dylib
SET(CMAKE_FIND_LIBRARY_SUFFIXES .${LVERS}.dylib)
+ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES FreeBSD)
+# FreeBSD has always .so installed.
+SET(CMAKE_FIND_LIBRARY_SUFFIXES .so)
ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES Windows)
# Windows doesn't support finding shared libraries by version.
ELSE()
@@ -95,8 +98,10 @@ GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRA
INCLUDE(CheckFunctionExists)
INCLUDE(CheckCSourceCompiles)
+INCLUDE(CheckLibraryExists)
CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
+CHECK_LIBRARY_EXISTS(dl dlopen NEED_LINK_DL)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
set_find_shared_library_version(1)
@@ -159,6 +164,9 @@ add_dual_library(hadoop
${D}/util/NativeCrc32.c
${D}/util/bulk_crc32.c
)
+if (NEED_LINK_DL)
+ set(LIB_DL dl)
+endif (NEED_LINK_DL)
IF (${CMAKE_SYSTEM_NAME} MATCHES Linux)
#
@@ -171,7 +179,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES Linux
ENDIF()
target_link_dual_libraries(hadoop
-dl
+${LIB_DL}
${JAVA_JVM_LIBRARY}
)
SET(LIBHADOOP_VERSION 1.0.0)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java?rev=1401321r1=1401320r2=1401321view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
Tue Oct 23 15:32:25 2012
@@ -43,7 +43,8 @@ public class HardLink {
OS_TYPE_UNIX,
OS_TYPE_WINXP,
OS_TYPE_SOLARIS,
-OS_TYPE_MAC
+OS_TYPE_MAC,
+OS_TYPE_FREEBSD
}
public static OSType osType;
@@ -63,7 +64,7 @@ public class HardLink {
getHardLinkCommand = new HardLinkCGUnix();
//override getLinkCountCommand for the particular Unix variant
//Linux is already set as the default - {stat,-c%h, null}
- if (osType == OSType.OS_TYPE_MAC) {
+ if (osType == OSType.OS_TYPE_MAC || osType == OSType.OS_TYPE_FREEBSD) {
String
svn commit: r1401323 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/ src/main/java/org/apache/hadoop/fs/ src/main/native/src/org/apache/hadoop/io/nativeio/ src/main/
Author: bobby
Date: Tue Oct 23 15:37:38 2012
New Revision: 1401323
URL: http://svn.apache.org/viewvc?rev=1401323view=rev
Log:
svn merge -c 1401321 FIXES: HADOOP-8811. Compile hadoop native library in
FreeBSD (Radim Kolar via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/getGroup.c
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1401323r1=1401322r2=1401323view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 23 15:37:38 2012
@@ -824,6 +824,9 @@ Release 0.23.5 - UNRELEASED
HADOOP-8906. paths with multiple globs are unreliable. (Daryn Sharp via
jlowe)
+HADOOP-8811. Compile hadoop native library in FreeBSD (Radim Kolar via
+bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1401323r1=1401322r2=1401323view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/CMakeLists.txt
Tue Oct 23 15:37:38 2012
@@ -67,6 +67,9 @@ macro(set_find_shared_library_version LV
IF(${CMAKE_SYSTEM_NAME} MATCHES Darwin)
# Mac OS uses .dylib
SET(CMAKE_FIND_LIBRARY_SUFFIXES .${LVERS}.dylib)
+ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES FreeBSD)
+# FreeBSD has always .so installed.
+SET(CMAKE_FIND_LIBRARY_SUFFIXES .so)
ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES Windows)
# Windows doesn't support finding shared libraries by version.
ELSE()
@@ -95,8 +98,10 @@ GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRA
INCLUDE(CheckFunctionExists)
INCLUDE(CheckCSourceCompiles)
+INCLUDE(CheckLibraryExists)
CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
+CHECK_LIBRARY_EXISTS(dl dlopen NEED_LINK_DL)
SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
set_find_shared_library_version(1)
@@ -159,6 +164,9 @@ add_dual_library(hadoop
${D}/util/NativeCrc32.c
${D}/util/bulk_crc32.c
)
+if (NEED_LINK_DL)
+ set(LIB_DL dl)
+endif (NEED_LINK_DL)
IF (${CMAKE_SYSTEM_NAME} MATCHES Linux)
#
@@ -171,7 +179,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES Linux
ENDIF()
target_link_dual_libraries(hadoop
-dl
+${LIB_DL}
${JAVA_JVM_LIBRARY}
)
SET(LIBHADOOP_VERSION 1.0.0)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java?rev=1401323r1=1401322r2=1401323view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
Tue Oct 23 15:37:38 2012
@@ -43,7 +43,8 @@ public class HardLink {
OS_TYPE_UNIX,
OS_TYPE_WINXP,
OS_TYPE_SOLARIS,
-OS_TYPE_MAC
+OS_TYPE_MAC,
+OS_TYPE_FREEBSD
}
public static OSType osType;
@@ -63,7 +64,7 @@ public class HardLink {
getHardLinkCommand = new
svn commit: r1401325 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java src/test/java/org/apache/hadoop/fs/TestLocalF
Author: bobby
Date: Tue Oct 23 15:47:26 2012
New Revision: 1401325
URL: http://svn.apache.org/viewvc?rev=1401325view=rev
Log:
HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename contains
a colon (jlowe via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1401325r1=1401324r2=1401325view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct
23 15:47:26 2012
@@ -1097,6 +1097,9 @@ Release 0.23.5 - UNRELEASED
HADOOP-8811. Compile hadoop native library in FreeBSD (Radim Kolar via
bobby)
+HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename
+contains a colon (jlowe via bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1401325r1=1401324r2=1401325view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Tue Oct 23 15:47:26 2012
@@ -350,7 +350,7 @@ public class RawLocalFileSystem extends
new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
}
-String[] names = localf.list();
+File[] names = localf.listFiles();
if (names == null) {
return null;
}
@@ -358,7 +358,7 @@ public class RawLocalFileSystem extends
int j = 0;
for (int i = 0; i names.length; i++) {
try {
-results[j] = getFileStatus(new Path(f, names[i]));
+results[j] = getFileStatus(new Path(names[i].getAbsolutePath()));
j++;
} catch (FileNotFoundException e) {
// ignore the files not found since the dir list may have have changed
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1401325r1=1401324r2=1401325view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Tue Oct 23 15:47:26 2012
@@ -249,6 +249,7 @@ public class TestLocalFileSystem {
assertEquals(1, fileSchemeCount);
}
+ @Test
public void testHasFileDescriptor() throws IOException {
Configuration conf = new Configuration();
LocalFileSystem fs = FileSystem.getLocal(conf);
@@ -258,4 +259,17 @@ public class TestLocalFileSystem {
new RawLocalFileSystem().new LocalFSFileInputStream(path), 1024);
assertNotNull(bis.getFileDescriptor());
}
+
+ @Test
+ public void testListStatusWithColons() throws IOException {
+Configuration conf = new Configuration();
+LocalFileSystem fs = FileSystem.getLocal(conf);
+File colonFile = new File(TEST_ROOT_DIR, foo:bar);
+colonFile.mkdirs();
+colonFile.createNewFile();
+FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
+assertEquals(Unexpected number of stats, 1, stats.length);
+assertEquals(Bad path from stat, colonFile.getAbsolutePath(),
+stats[0].getPath().toUri().getPath());
+ }
}
svn commit: r1401326 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java src/test/java/org/apache/hadoop/f
Author: bobby
Date: Tue Oct 23 15:48:26 2012
New Revision: 1401326
URL: http://svn.apache.org/viewvc?rev=1401326view=rev
Log:
svn merge -c 1401325 FIXES: HADOOP-8962. RawLocalFileSystem.listStatus fails
when a child filename contains a colon (jlowe via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1401326r1=1401325r2=1401326view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 23 15:48:26 2012
@@ -827,6 +827,9 @@ Release 0.23.5 - UNRELEASED
HADOOP-8811. Compile hadoop native library in FreeBSD (Radim Kolar via
bobby)
+HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename
+contains a colon (jlowe via bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1401326r1=1401325r2=1401326view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Tue Oct 23 15:48:26 2012
@@ -344,7 +344,7 @@ public class RawLocalFileSystem extends
new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) };
}
-String[] names = localf.list();
+File[] names = localf.listFiles();
if (names == null) {
return null;
}
@@ -352,7 +352,7 @@ public class RawLocalFileSystem extends
int j = 0;
for (int i = 0; i names.length; i++) {
try {
-results[j] = getFileStatus(new Path(f, names[i]));
+results[j] = getFileStatus(new Path(names[i].getAbsolutePath()));
j++;
} catch (FileNotFoundException e) {
// ignore the files not found since the dir list may have have changed
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1401326r1=1401325r2=1401326view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Tue Oct 23 15:48:26 2012
@@ -249,6 +249,7 @@ public class TestLocalFileSystem {
assertEquals(1, fileSchemeCount);
}
+ @Test
public void testHasFileDescriptor() throws IOException {
Configuration conf = new Configuration();
LocalFileSystem fs = FileSystem.getLocal(conf);
@@ -258,4 +259,17 @@ public class TestLocalFileSystem {
new RawLocalFileSystem().new LocalFSFileInputStream(path), 1024);
assertNotNull(bis.getFileDescriptor());
}
+
+ @Test
+ public void testListStatusWithColons() throws IOException {
+Configuration conf = new Configuration();
+LocalFileSystem fs = FileSystem.getLocal(conf);
+File colonFile = new File(TEST_ROOT_DIR, foo:bar);
+colonFile.mkdirs();
+colonFile.createNewFile();
+FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
+assertEquals(Unexpected number of stats, 1, stats.length);
+assertEquals(Bad path from stat, colonFile.getAbsolutePath(),
+stats[0].getPath().toUri().getPath());
+ }
}
svn commit: r1401328 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java src/test/java/org/apache/hadoo
Author: bobby
Date: Tue Oct 23 15:52:41 2012
New Revision: 1401328
URL: http://svn.apache.org/viewvc?rev=1401328view=rev
Log:
svn merge -c 1401325 FIXES: HADOOP-8962. RawLocalFileSystem.listStatus fails
when a child filename contains a colon (jlowe via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1401328r1=1401327r2=1401328view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 23 15:52:41 2012
@@ -47,6 +47,9 @@ Release 0.23.5 - UNRELEASED
HADOOP-8906. paths with multiple globs are unreliable. (Daryn Sharp via
jlowe)
+HADOOP-8962. RawLocalFileSystem.listStatus fails when a child filename
+contains a colon (jlowe via bobby)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1401328r1=1401327r2=1401328view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Tue Oct 23 15:52:41 2012
@@ -319,7 +319,7 @@ public class RawLocalFileSystem extends
new RawLocalFileStatus(localf, getDefaultBlockSize(), this) };
}
-String[] names = localf.list();
+File[] names = localf.listFiles();
if (names == null) {
return null;
}
@@ -327,7 +327,7 @@ public class RawLocalFileSystem extends
int j = 0;
for (int i = 0; i names.length; i++) {
try {
-results[j] = getFileStatus(new Path(f, names[i]));
+results[j] = getFileStatus(new Path(names[i].getAbsolutePath()));
j++;
} catch (FileNotFoundException e) {
// ignore the files not found since the dir list may have have changed
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java?rev=1401328r1=1401327r2=1401328view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
Tue Oct 23 15:52:41 2012
@@ -248,4 +248,17 @@ public class TestLocalFileSystem {
}
assertEquals(1, fileSchemeCount);
}
+
+ @Test
+ public void testListStatusWithColons() throws IOException {
+Configuration conf = new Configuration();
+LocalFileSystem fs = FileSystem.getLocal(conf);
+File colonFile = new File(TEST_ROOT_DIR, foo:bar);
+colonFile.mkdirs();
+colonFile.createNewFile();
+FileStatus[] stats = fs.listStatus(new Path(TEST_ROOT_DIR));
+assertEquals(Unexpected number of stats, 1, stats.length);
+assertEquals(Bad path from stat, colonFile.getAbsolutePath(),
+stats[0].getPath().toUri().getPath());
+ }
}
svn commit: r1400224 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Fri Oct 19 18:50:44 2012 New Revision: 1400224 URL: http://svn.apache.org/viewvc?rev=1400224view=rev Log: svn merge -c 1394881 FIXES: HADOOP-8889. Upgrade to Surefire 2.12.3. Contributed by Todd Lipcon. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1400224r1=1400223r2=1400224view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct 19 18:50:44 2012 @@ -23,6 +23,8 @@ Release 0.23.5 - UNRELEASED HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via bobby) +HADOOP-8889. Upgrade to Surefire 2.12.3 (todd) + OPTIMIZATIONS HADOOP-8819. Incorrectly is used instead of in some file system
svn commit: r1399426 - in /hadoop/common/trunk: hadoop-project/pom.xml pom.xml
Author: bobby
Date: Wed Oct 17 20:49:44 2012
New Revision: 1399426
URL: http://svn.apache.org/viewvc?rev=1399426view=rev
Log:
HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via bobby)
Modified:
hadoop/common/trunk/hadoop-project/pom.xml
hadoop/common/trunk/pom.xml
Modified: hadoop/common/trunk/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-project/pom.xml?rev=1399426r1=1399425r2=1399426view=diff
==
--- hadoop/common/trunk/hadoop-project/pom.xml (original)
+++ hadoop/common/trunk/hadoop-project/pom.xml Wed Oct 17 20:49:44 2012
@@ -656,12 +656,6 @@
scopecompile/scope
/dependency
dependency
-groupIdcom.cenqua.clover/groupId
-artifactIdclover/artifactId
-!-- Use the version needed by maven-clover-plugin --
-version3.0.2/version
- /dependency
- dependency
groupIdorg.hsqldb/groupId
artifactIdhsqldb/artifactId
version2.0.0/version
@@ -731,11 +725,6 @@
version2.3.2/version
/plugin
plugin
- groupIdcom.atlassian.maven.plugins/groupId
- artifactIdmaven-clover2-plugin/artifactId
- version3.0.5/version
-/plugin
-plugin
groupIdorg.apache.maven.plugins/groupId
artifactIdmaven-checkstyle-plugin/artifactId
version2.6/version
@@ -909,54 +898,6 @@
build.platformMac_OS_X-${sun.arch.data.model}/build.platform
/properties
/profile
-
-profile
- idclover/id
- activation
-activeByDefaultfalse/activeByDefault
-property
- nameclover/name
-/property
- /activation
- properties
-
cloverLicenseLocation${user.home}/.clover.license/cloverLicenseLocation
-
cloverDatabase${project.build.directory}/clover/hadoop-coverage.db/cloverDatabase
- /properties
- build
-plugins
- plugin
-groupIdcom.atlassian.maven.plugins/groupId
-artifactIdmaven-clover2-plugin/artifactId
-configuration
- includesAllSourceRootstrue/includesAllSourceRoots
- includesTestSourceRootstrue/includesTestSourceRoots
- licenseLocation${cloverLicenseLocation}/licenseLocation
- cloverDatabase${cloverDatabase}/cloverDatabase
- targetPercentage50%/targetPercentage
-
outputDirectory${project.build.directory}/clover/outputDirectory
- generateHtmltrue/generateHtml
- generateXmltrue/generateXml
-/configuration
-executions
- execution
-idclover-setup/id
-phaseprocess-sources/phase
-goals
- goalsetup/goal
-/goals
- /execution
- execution
-idclover/id
-phasetest/phase
-goals
- goalclover/goal
-/goals
- /execution
-/executions
- /plugin
-/plugins
- /build
-/profile
profile
idtest-patch/id
activation
Modified: hadoop/common/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/pom.xml?rev=1399426r1=1399425r2=1399426view=diff
==
--- hadoop/common/trunk/pom.xml (original)
+++ hadoop/common/trunk/pom.xml Wed Oct 17 20:49:44 2012
@@ -23,6 +23,17 @@ xsi:schemaLocation=http://maven.apache.
nameApache Hadoop Main/name
packagingpom/packaging
+ dependencyManagement
+dependencies
+ dependency
+groupIdcom.cenqua.clover/groupId
+artifactIdclover/artifactId
+!-- Use the version needed by maven-clover-plugin --
+version3.0.2/version
+ /dependency
+/dependencies
+ /dependencyManagement
+
distributionManagement
repository
idapache.staging.https/id
@@ -243,6 +254,11 @@ xsi:schemaLocation=http://maven.apache.
/lifecycleMappingMetadata
/configuration
/plugin
+plugin
+ groupIdcom.atlassian.maven.plugins/groupId
+ artifactIdmaven-clover2-plugin/artifactId
+ version3.0.5/version
+/plugin
/plugins
/pluginManagement
@@ -483,6 +499,52 @@ xsi:schemaLocation=http://maven.apache.
/plugins
/build
/profile
-
+profile
+ idclover/id
+ activation
+activeByDefaultfalse/activeByDefault
+property
+ nameclover/name
+/property
+ /activation
+ properties
+
cloverLicenseLocation${user.home}/.clover.license/cloverLicenseLocation
+
cloverDatabase${project.build.directory}/clover/hadoop-coverage.db/cloverDatabase
+ /properties
+ build
+plugins
+ plugin
svn commit: r1399426 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Wed Oct 17 20:49:44 2012 New Revision: 1399426 URL: http://svn.apache.org/viewvc?rev=1399426view=rev Log: HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via bobby) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1399426r1=1399425r2=1399426view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 17 20:49:44 2012 @@ -1069,9 +1069,13 @@ Release 0.23.5 - UNRELEASED NEW FEATURES IMPROVEMENTS + HADOOP-8923. JNI-based user-group mapping modules can be too chatty on lookup failures. (Kihwal Lee via suresh) +HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via +bobby) + OPTIMIZATIONS BUG FIXES
svn commit: r1399427 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Wed Oct 17 20:51:28 2012 New Revision: 1399427 URL: http://svn.apache.org/viewvc?rev=1399427view=rev Log: svn merge -c 1399426 FIXES: HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via bobby) Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1399427r1=1399426r2=1399427view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 17 20:51:28 2012 @@ -802,9 +802,13 @@ Release 0.23.5 - UNRELEASED NEW FEATURES IMPROVEMENTS + HADOOP-8923. JNI-based user-group mapping modules can be too chatty on lookup failures. (Kihwal Lee via suresh) +HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via +bobby) + OPTIMIZATIONS BUG FIXES
svn commit: r1399428 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Wed Oct 17 20:55:31 2012 New Revision: 1399428 URL: http://svn.apache.org/viewvc?rev=1399428view=rev Log: HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via bobby) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1399428r1=1399427r2=1399428view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 17 20:55:31 2012 @@ -20,6 +20,9 @@ Release 0.23.5 - UNRELEASED HADOOP-8923. JNI-based user-group mapping modules can be too chatty on lookup failures. (Kihwal Lee via suresh) +HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via +bobby) + OPTIMIZATIONS HADOOP-8819. Incorrectly is used instead of in some file system
svn commit: r1399428 - in /hadoop/common/branches/branch-0.23: hadoop-project/pom.xml pom.xml
Author: bobby
Date: Wed Oct 17 20:55:31 2012
New Revision: 1399428
URL: http://svn.apache.org/viewvc?rev=1399428view=rev
Log:
HADOOP-8930. Cumulative code coverage calculation (Andrey Klochkov via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-project/pom.xml
hadoop/common/branches/branch-0.23/pom.xml
Modified: hadoop/common/branches/branch-0.23/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-project/pom.xml?rev=1399428r1=1399427r2=1399428view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-project/pom.xml Wed Oct 17
20:55:31 2012
@@ -615,13 +615,6 @@
artifactIdjson-simple/artifactId
version1.1/version
/dependency
-
- dependency
-groupIdcom.cenqua.clover/groupId
-artifactIdclover/artifactId
-version3.0.2/version
- /dependency
-
dependency
groupIdorg.apache.zookeeper/groupId
artifactIdzookeeper/artifactId
@@ -714,11 +707,6 @@
version2.3.2/version
/plugin
plugin
- groupIdcom.atlassian.maven.plugins/groupId
- artifactIdmaven-clover2-plugin/artifactId
- version3.0.5/version
-/plugin
-plugin
groupIdorg.apache.maven.plugins/groupId
artifactIdmaven-checkstyle-plugin/artifactId
version2.6/version
@@ -928,53 +916,6 @@
build.platformMac_OS_X-${sun.arch.data.model}/build.platform
/properties
/profile
-
-profile
- idclover/id
- activation
-activeByDefaultfalse/activeByDefault
-property
- nameclover/name
-/property
- /activation
- properties
-
cloverLicenseLocation${user.home}/.clover.license/cloverLicenseLocation
-
cloverDatabase${project.build.directory}/clover/hadoop-coverage.db/cloverDatabase
- /properties
- build
-plugins
- plugin
-groupIdcom.atlassian.maven.plugins/groupId
-artifactIdmaven-clover2-plugin/artifactId
-configuration
- includesTestSourceRootstrue/includesTestSourceRoots
- licenseLocation${cloverLicenseLocation}/licenseLocation
- cloverDatabase${cloverDatabase}/cloverDatabase
- targetPercentage50%/targetPercentage
-
outputDirectory${project.build.directory}/clover/outputDirectory
- generateHtmltrue/generateHtml
- generateXmltrue/generateXml
-/configuration
-executions
- execution
-idsetup/id
-phasegenerate-sources/phase
-goals
- goalsetup/goal
-/goals
- /execution
- execution
-idclover/id
-phasetest/phase
-goals
- goalclover/goal
-/goals
- /execution
-/executions
- /plugin
-/plugins
- /build
-/profile
profile
idtest-patch/id
activation
Modified: hadoop/common/branches/branch-0.23/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/pom.xml?rev=1399428r1=1399427r2=1399428view=diff
==
--- hadoop/common/branches/branch-0.23/pom.xml (original)
+++ hadoop/common/branches/branch-0.23/pom.xml Wed Oct 17 20:55:31 2012
@@ -24,6 +24,16 @@
nameApache Hadoop Main/name
packagingpom/packaging
+ dependencyManagement
+dependencies
+ dependency
+groupIdcom.cenqua.clover/groupId
+artifactIdclover/artifactId
+version3.0.2/version
+ /dependency
+/dependencies
+ /dependencyManagement
+
distributionManagement
repository
idapache.staging.https/id
@@ -137,6 +147,11 @@
/dependency
/dependencies
/plugin
+plugin
+ groupIdcom.atlassian.maven.plugins/groupId
+ artifactIdmaven-clover2-plugin/artifactId
+ version3.0.5/version
+/plugin
/plugins
/pluginManagement
@@ -365,6 +380,52 @@
/plugins
/build
/profile
-
+profile
+ idclover/id
+ activation
+activeByDefaultfalse/activeByDefault
+property
+ nameclover/name
+/property
+ /activation
+ properties
+
cloverLicenseLocation${user.home}/.clover.license/cloverLicenseLocation
+
cloverDatabase${project.build.directory}/clover/hadoop-coverage.db/cloverDatabase
+ /properties
+ build
+plugins
+ plugin
+groupIdcom.atlassian.maven.plugins/groupId
+artifactIdmaven-clover2-plugin/artifactId
+configuration
svn commit: r1398901 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
Author: bobby
Date: Tue Oct 16 17:52:01 2012
New Revision: 1398901
URL: http://svn.apache.org/viewvc?rev=1398901view=rev
Log:
HDFS-4016. back-port HDFS-3582 to branch-0.23 (Ivan A. Veselovsky via bobby)
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java?rev=1398901view=auto
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
(added)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ExitUtil.java
Tue Oct 16 17:52:01 2012
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Facilitates hooking process termination for tests and debugging.
+ */
+@InterfaceAudience.LimitedPrivate({HDFS, MapReduce})
+@InterfaceStability.Unstable
+public final class ExitUtil {
+ private final static Log LOG = LogFactory.getLog(ExitUtil.class.getName());
+ private static volatile boolean systemExitDisabled = false;
+ private static volatile boolean terminateCalled = false;
+
+ public static class ExitException extends RuntimeException {
+private static final long serialVersionUID = 1L;
+public final int status;
+
+public ExitException(int status, String msg) {
+ super(msg);
+ this.status = status;
+}
+ }
+
+ /**
+ * Disable the use of System.exit for testing.
+ */
+ public static void disableSystemExit() {
+systemExitDisabled = true;
+ }
+
+ /**
+ * @return true if terminate has been called
+ */
+ public static boolean terminateCalled() {
+return terminateCalled;
+ }
+
+ /**
+ * Terminate the current process. Note that terminate is the *only* method
+ * that should be used to terminate the daemon processes.
+ * @param status exit code
+ * @param msg message used to create the ExitException
+ * @throws ExitException if System.exit is disabled for test purposes
+ */
+ public static void terminate(int status, String msg) throws ExitException {
+LOG.info(Exiting with status + status);
+terminateCalled = true;
+if (systemExitDisabled) {
+ throw new ExitException(status, msg);
+}
+System.exit(status);
+ }
+
+ /**
+ * Like {@link terminate(int, String)} without a message.
+ * @param status
+ * @throws ExitException
+ */
+ public static void terminate(int status) throws ExitException {
+terminate(status, ExitException);
+ }
+}
\ No newline at end of file
svn commit: r1398904 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java src/test/java/org/apache/hadoop/jmx/TestJMXJsonS
Author: bobby
Date: Tue Oct 16 18:00:01 2012
New Revision: 1398904
URL: http://svn.apache.org/viewvc?rev=1398904view=rev
Log:
HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
javascript in browser dashboard (Damien Hardy via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1398904r1=1398903r2=1398904view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct
16 18:00:01 2012
@@ -320,6 +320,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
+HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
+javascript in browser dashboard (Damien Hardy via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1398904r1=1398903r2=1398904view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
Tue Oct 16 18:00:01 2012
@@ -113,12 +113,17 @@ import org.codehaus.jackson.JsonGenerato
* All other objects will be converted to a string and output as such.
*
* The bean's name and modelerType will be returned for all beans.
+ *
+ * Optional paramater callback should be used to deliver JSONP response.
+ *
*/
public class JMXJsonServlet extends HttpServlet {
private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
private static final long serialVersionUID = 1L;
+ private static final String CALLBACK_PARAM = callback;
+
/**
* MBean server.
*/
@@ -154,11 +159,22 @@ public class JMXJsonServlet extends Http
return;
}
JsonGenerator jg = null;
+ String jsonpcb = null;
+ PrintWriter writer = null;
try {
-response.setContentType(application/json; charset=utf8);
+writer = response.getWriter();
+
+// callback parameter implies JSONP outpout
+jsonpcb = request.getParameter(CALLBACK_PARAM);
+if (jsonpcb != null) {
+ response.setContentType(application/javascript; charset=utf8);
+ writer.write(jsonpcb + ();
+} else {
+ response.setContentType(application/json; charset=utf8);
+}
-PrintWriter writer = response.getWriter();
jg = jsonFactory.createJsonGenerator(writer);
+jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
jg.useDefaultPrettyPrinter();
jg.writeStartObject();
@@ -188,6 +204,12 @@ public class JMXJsonServlet extends Http
if (jg != null) {
jg.close();
}
+if (jsonpcb != null) {
+ writer.write(););
+}
+if (writer != null) {
+ writer.close();
+}
}
} catch (IOException e) {
LOG.error(Caught an exception while processing JMX request, e);
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java?rev=1398904r1=1398903r2=1398904view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
Tue Oct 16 18:00:01 2012
@@ -78,5 +78,29 @@ public class TestJMXJsonServlet extends
/jmx?get=java.lang:type=Memory::));
LOG.info(/jmx RESULT: +result);
assertReFind(\ERROR\, result);
+
+// test to get JSONP result
+result = readOutput(new URL(baseUrl,
/jmx?qry=java.lang:type=Memorycallback=mycallback1));
+LOG.info(/jmx?qry=java.lang:type=Memorycallback
svn commit: r1399001 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java src/test/java/org/apache/hadoop/jmx/
Author: bobby
Date: Tue Oct 16 21:40:53 2012
New Revision: 1399001
URL: http://svn.apache.org/viewvc?rev=1399001view=rev
Log:
HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
javascript in browser (Damien Hardy via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/jmx/TestJMXJsonServlet.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1399001r1=1399000r2=1399001view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 16 21:40:53 2012
@@ -51,6 +51,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8929. Add toString, other improvements for SampleQuantiles (todd)
+HADOOP-8922. Provide alternate JSONP output for JMXJsonServlet to allow
+javascript in browser (Damien Hardy via bobby)
+
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1399001r1=1399000r2=1399001view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/jmx/JMXJsonServlet.java
Tue Oct 16 21:40:53 2012
@@ -113,6 +113,9 @@ import org.codehaus.jackson.JsonGenerato
* All other objects will be converted to a string and output as such.
*
* The bean's name and modelerType will be returned for all beans.
+ *
+ * Optional paramater callback should be used to deliver JSONP response.
+ *
*/
public class JMXJsonServlet extends HttpServlet {
private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
@@ -120,6 +123,8 @@ public class JMXJsonServlet extends Http
private static final long serialVersionUID = 1L;
// - Instance Variables
+ private static final String CALLBACK_PARAM = callback;
+
/**
* MBean server.
*/
@@ -145,20 +150,33 @@ public class JMXJsonServlet extends Http
*/
@Override
public void doGet(HttpServletRequest request, HttpServletResponse response) {
+String jsonpcb = null;
+PrintWriter writer = null;
try {
if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
request, response)) {
return;
}
+
+ JsonGenerator jg = null;
- response.setContentType(application/json; charset=utf8);
-
- PrintWriter writer = response.getWriter();
+ writer = response.getWriter();
+
+ // callback parameter implies JSONP outpout
+ jsonpcb = request.getParameter(CALLBACK_PARAM);
+ if (jsonpcb != null) {
+response.setContentType(application/javascript; charset=utf8);
+writer.write(jsonpcb + ();
+ } else {
+response.setContentType(application/json; charset=utf8);
+ }
JsonFactory jsonFactory = new JsonFactory();
- JsonGenerator jg = jsonFactory.createJsonGenerator(writer);
+ jg = jsonFactory.createJsonGenerator(writer);
+ jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
jg.useDefaultPrettyPrinter();
jg.writeStartObject();
+
if (mBeanServer == null) {
jg.writeStringField(result, ERROR);
jg.writeStringField(message, No MBeanServer could be found);
@@ -183,6 +201,7 @@ public class JMXJsonServlet extends Http
response);
jg.close();
return;
+
}
// query per mbean
@@ -199,6 +218,13 @@ public class JMXJsonServlet extends Http
} catch ( MalformedObjectNameException e ) {
LOG.error(Caught an exception while processing JMX request, e);
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+} finally {
+ if (jsonpcb != null) {
+ writer.write(););
+ }
+ if (writer != null) {
+writer.close();
+ }
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java
svn commit: r1399005 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/util/ src/test/java/org/apache/hadoop/util/
Author: bobby
Date: Tue Oct 16 21:52:10 2012
New Revision: 1399005
URL: http://svn.apache.org/viewvc?rev=1399005view=rev
Log:
HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static data
(Gopal V via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1399005r1=1399004r2=1399005view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Oct
16 21:52:10 2012
@@ -330,6 +330,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
via atm)
+HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
+data (Gopal V via bobby)
+
BUG FIXES
HADOOP-8795. BASH tab completion doesn't look in PATH, assumes path to
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java?rev=1399005r1=1399004r2=1399005view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
Tue Oct 16 21:52:10 2012
@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Ch
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc = 8);
- int c2 = b[off++] ^ (localCrc = 8);
- int c3 = b[off++] ^ (localCrc = 8);
- localCrc = (T8_7[c0 0xff] ^ T8_6[c1 0xff])
- ^ (T8_5[c2 0xff] ^ T8_4[c3 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) 0xff;
+ final int c1 =(b[off+1] ^ (localCrc = 8)) 0xff;
+ final int c2 =(b[off+2] ^ (localCrc = 8)) 0xff;
+ final int c3 =(b[off+3] ^ (localCrc = 8)) 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] 0xff;
+ final int c5 = b[off+5] 0xff;
+ final int c6 = b[off+6] 0xff;
+ final int c7 = b[off+7] 0xff;
- localCrc ^= (T8_3[b[off++] 0xff] ^ T8_2[b[off++] 0xff])
- ^ (T8_1[b[off++] 0xff] ^ T8_0[b[off++] 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
-while(len 0) {
- localCrc = (localCrc 8) ^ T8_0[(localCrc ^ b[off++]) 0xff];
- len--;
+
+/* loop unroll - duff's device style */
+switch(len) {
+ case 7: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 6: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 5: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 4: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 3: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 2: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 1: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ default:
+/* nothing */
}
// Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Ch
@Override
final public void update(int b) {
-crc = (crc 8) ^ T8_0[(crc ^ b) 0xff];
+crc = (crc 8) ^ T[T8_0_start + ((crc ^ b) 0xff)];
}
/*
* CRC-32 lookup tables generated by the polynomial 0xEDB88320.
* See also TestPureJavaCrc32.Table.
*/
- private static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256;
+ private static final int T8_2_start = 2*256;
+ private static final int T8_3_start = 3*256;
+ private static final int T8_4_start = 4*256;
+ private static final int T8_5_start = 5*256;
+ private static final int T8_6_start = 6*256
svn commit: r1399006 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/util/ src/test/java/org/apache/hadoop/util/
Author: bobby
Date: Tue Oct 16 21:55:01 2012
New Revision: 1399006
URL: http://svn.apache.org/viewvc?rev=1399006view=rev
Log:
svn merge -c 1399005. FIXES: HADOOP-8926. hadoop.util.PureJavaCrc32 cache
hit-ratio is low for static data (Gopal V via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1399006r1=1399005r2=1399006view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Oct 16 21:55:01 2012
@@ -59,6 +59,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
via atm)
+HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
+data (Gopal V via bobby)
+
BUG FIXES
HADOOP-8795. BASH tab completion doesn't look in PATH, assumes path to
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java?rev=1399006r1=1399005r2=1399006view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
Tue Oct 16 21:55:01 2012
@@ -59,22 +59,38 @@ public class PureJavaCrc32 implements Ch
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
+
while(len 7) {
- int c0 = b[off++] ^ localCrc;
- int c1 = b[off++] ^ (localCrc = 8);
- int c2 = b[off++] ^ (localCrc = 8);
- int c3 = b[off++] ^ (localCrc = 8);
- localCrc = (T8_7[c0 0xff] ^ T8_6[c1 0xff])
- ^ (T8_5[c2 0xff] ^ T8_4[c3 0xff]);
+ final int c0 =(b[off+0] ^ localCrc) 0xff;
+ final int c1 =(b[off+1] ^ (localCrc = 8)) 0xff;
+ final int c2 =(b[off+2] ^ (localCrc = 8)) 0xff;
+ final int c3 =(b[off+3] ^ (localCrc = 8)) 0xff;
+ localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
+ ^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
+
+ final int c4 = b[off+4] 0xff;
+ final int c5 = b[off+5] 0xff;
+ final int c6 = b[off+6] 0xff;
+ final int c7 = b[off+7] 0xff;
- localCrc ^= (T8_3[b[off++] 0xff] ^ T8_2[b[off++] 0xff])
- ^ (T8_1[b[off++] 0xff] ^ T8_0[b[off++] 0xff]);
+ localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
+ ^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
+ off += 8;
len -= 8;
}
-while(len 0) {
- localCrc = (localCrc 8) ^ T8_0[(localCrc ^ b[off++]) 0xff];
- len--;
+
+/* loop unroll - duff's device style */
+switch(len) {
+ case 7: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 6: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 5: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 4: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 3: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 2: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ case 1: localCrc = (localCrc 8) ^ T[T8_0_start + ((localCrc ^
b[off++]) 0xff)];
+ default:
+/* nothing */
}
// Publish crc out to object
@@ -83,14 +99,24 @@ public class PureJavaCrc32 implements Ch
@Override
final public void update(int b) {
-crc = (crc 8) ^ T8_0[(crc ^ b) 0xff];
+crc = (crc 8) ^ T[T8_0_start + ((crc ^ b) 0xff)];
}
/*
* CRC-32 lookup tables generated by the polynomial 0xEDB88320.
* See also TestPureJavaCrc32.Table.
*/
- private static final int[] T8_0 = new int[] {
+ private static final int T8_0_start = 0*256;
+ private static final int T8_1_start = 1*256;
+ private static final int T8_2_start = 2*256;
+ private static
svn commit: r1398515 - in /hadoop/common/tags: release-0.23.4-rc2/ release-0.23.4/
Author: bobby Date: Mon Oct 15 21:04:04 2012 New Revision: 1398515 URL: http://svn.apache.org/viewvc?rev=1398515view=rev Log: Moveing tag for 0.23.4 release. Added: hadoop/common/tags/release-0.23.4/ (props changed) - copied from r1398513, hadoop/common/tags/release-0.23.4-rc2/ Removed: hadoop/common/tags/release-0.23.4-rc2/ Propchange: hadoop/common/tags/release-0.23.4/ -- --- svn:ignore (added) +++ svn:ignore Mon Oct 15 21:04:04 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/tags/release-0.23.4/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Oct 15 21:04:04 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1395599 - /hadoop/common/tags/release-0.23.4-rc2/
Author: bobby Date: Mon Oct 8 14:58:48 2012 New Revision: 1395599 URL: http://svn.apache.org/viewvc?rev=1395599view=rev Log: Spinning rc2 of 0.23.4 Added: hadoop/common/tags/release-0.23.4-rc2/ (props changed) - copied from r1395579, hadoop/common/branches/branch-0.23.4/ Propchange: hadoop/common/tags/release-0.23.4-rc2/ -- --- svn:ignore (added) +++ svn:ignore Mon Oct 8 14:58:48 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/tags/release-0.23.4-rc2/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Oct 8 14:58:48 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1395601 - in /hadoop/common/tags: release-0.23.4-rc0/ release-0.23.4-rc1/
Author: bobby Date: Mon Oct 8 15:02:50 2012 New Revision: 1395601 URL: http://svn.apache.org/viewvc?rev=1395601view=rev Log: Removing old RCs Removed: hadoop/common/tags/release-0.23.4-rc0/ hadoop/common/tags/release-0.23.4-rc1/
svn commit: r1392368 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/bin/hadoop
Author: bobby
Date: Mon Oct 1 14:53:50 2012
New Revision: 1392368
URL: http://svn.apache.org/viewvc?rev=1392368view=rev
Log:
svn merge -c 1391780 FIXES: HADOOP-8386. hadoop script doesn't work if 'cd'
prints to stdout (default behavior in Ubuntu). Contributed by Christopher
Berner and Andy Isaacson. (harsh)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392368r1=1392367r2=1392368view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Oct 1 14:53:50 2012
@@ -12,6 +12,10 @@ Release 0.23.5 - UNRELEASED
BUG FIXES
+HADOOP-8386. hadoop script doesn't work if 'cd' prints to stdout
+(default behavior in some bash setups (esp. Ubuntu))
+(Chiristopher Berner and Andy Isaacson via harsh)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1392368r1=1392367r2=1392368view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/bin/hadoop
Mon Oct 1 14:53:50 2012
@@ -19,7 +19,7 @@
bin=`which $0`
bin=`dirname ${bin}`
-bin=`cd $bin; pwd`
+bin=`cd $bin /dev/null; pwd`
DEFAULT_LIBEXEC_DIR=$bin/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
svn commit: r1392437 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/docs/src/documentation/content/xdocs/file_system_shell.xml
Author: bobby Date: Mon Oct 1 16:47:09 2012 New Revision: 1392437 URL: http://svn.apache.org/viewvc?rev=1392437view=rev Log: svn merge -c 1390109 FIXES: HADOOP-8791. Fix rm command documentation to indicte it deletes files and not directories. Contributed by Jing Zhao. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392437r1=1392436r2=1392437view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 16:47:09 2012 @@ -16,6 +16,9 @@ Release 0.23.5 - UNRELEASED (default behavior in some bash setups (esp. Ubuntu)) (Chiristopher Berner and Andy Isaacson via harsh) +HADOOP-8791. Fix rm command documentation to indicte it deletes +files and not directories. (Jing Zhao via suresh) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml?rev=1392437r1=1392436r2=1392437view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/file_system_shell.xml Mon Oct 1 16:47:09 2012 @@ -418,15 +418,15 @@ codeUsage: hdfs dfs -rm [-skipTrash] URI [URI #x2026;] /code /p p - Delete files specified as args. Only deletes non empty directory and files. If the code-skipTrash/code option + Delete files specified as args. Only deletes files. If the code-skipTrash/code option is specified, the trash, if enabled, will be bypassed and the specified file(s) deleted immediately. This can be useful when it is necessary to delete files from an over-quota directory. - Refer to rmr for recursive deletes.br/ + Use -rm -r or rmr for recursive deletes.br/ Example: /p ul li - code hdfs dfs -rm hdfs://nn.example.com/file /user/hadoop/emptydir /code + code hdfs dfs -rm hdfs://nn.example.com/file /code /li /ul pExit Code:/p @@ -442,7 +442,7 @@ p codeUsage: hdfs dfs -rmr [-skipTrash] URI [URI #x2026;]/code /p - pRecursive version of delete. If the code-skipTrash/code option + pRecursive version of delete. The rmr command recursively deletes the directory and any content under it. If the code-skipTrash/code option is specified, the trash, if enabled, will be bypassed and the specified file(s) deleted immediately. This can be useful when it is necessary to delete files from an over-quota directory.br/ Example:
svn commit: r1392458 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java src/test/java/org/apa
Author: bobby Date: Mon Oct 1 17:34:49 2012 New Revision: 1392458 URL: http://svn.apache.org/viewvc?rev=1392458view=rev Log: svn merge -c 1384627 FIXES: HADOOP-8755. Print thread dump when tests fail due to timeout. Contributed by Andrey Klochkov. Added: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java - copied unchanged from r1384627, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestTimedOutTestsListener.java hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TimedOutTestsListener.java - copied unchanged from r1384627, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TimedOutTestsListener.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392458r1=1392457r2=1392458view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 17:34:49 2012 @@ -8,6 +8,9 @@ Release 0.23.5 - UNRELEASED IMPROVEMENTS +HADOOP-8755. Print thread dump when tests fail due to timeout. (Andrey +Klochkov via atm) + OPTIMIZATIONS BUG FIXES
svn commit: r1392465 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ftp/ src/main/java/org/apache/hadoop/fs/s3/ src/main/java/org/a
Author: bobby
Date: Mon Oct 1 17:44:15 2012
New Revision: 1392465
URL: http://svn.apache.org/viewvc?rev=1392465view=rev
Log:
svn merge -c 1386451 FIXES: HADOOP-8819. Incorrectly is used instead of in
some file system implementations. Contributed by Brandon Li.
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392465r1=1392464r2=1392465view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Oct 1 17:44:15 2012
@@ -13,6 +13,9 @@ Release 0.23.5 - UNRELEASED
OPTIMIZATIONS
+HADOOP-8819. Incorrectly is used instead of in some file system
+implementations. (Brandon Li via suresh)
+
BUG FIXES
HADOOP-8386. hadoop script doesn't work if 'cd' prints to stdout
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java?rev=1392465r1=1392464r2=1392465view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
Mon Oct 1 17:44:15 2012
@@ -476,7 +476,7 @@ public class FTPFileSystem extends FileS
if (created) {
String parentDir = parent.toUri().getPath();
client.changeWorkingDirectory(parentDir);
-created = created client.makeDirectory(pathName);
+created = created client.makeDirectory(pathName);
}
} else if (isFile(client, absolute)) {
throw new IOException(String.format(
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java?rev=1392465r1=1392464r2=1392465view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java
Mon Oct 1 17:44:15 2012
@@ -73,7 +73,7 @@ public class FTPInputStream extends FSIn
if (byteRead = 0) {
pos++;
}
-if (stats != null byteRead = 0) {
+if (stats != null byteRead = 0) {
stats.incrementBytesRead(1);
}
return byteRead;
@@ -88,7 +88,7 @@ public class FTPInputStream extends FSIn
if (result 0) {
pos += result;
}
-if (stats != null result 0) {
+if (stats != null result 0) {
stats.incrementBytesRead(result);
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java?rev=1392465r1=1392464r2=1392465view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3InputStream.java
Mon Oct 1 17:44:15 2012
@@ -113,7 +113,7 @@ class S3InputStream extends FSInputStrea
svn commit: r1392492 - in /hadoop/common/branches/branch-0.23/hadoop-tools: hadoop-archives/src/test/java/org/apache/hadoop/tools/ hadoop-extras/src/test/java/org/apache/hadoop/tools/
Author: bobby
Date: Mon Oct 1 18:15:44 2012
New Revision: 1392492
URL: http://svn.apache.org/viewvc?rev=1392492view=rev
Log:
svn merge -c 1383494 FIXES: HADOOP-8789. Tests setLevel(Level.OFF) should be
Level.ERROR. Contributed by Andy Isaacson
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java?rev=1392492r1=1392491r2=1392492view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
Mon Oct 1 18:15:44 2012
@@ -52,11 +52,11 @@ public class TestHadoopArchives extends
{
((Log4JLogger)LogFactory.getLog(org.apache.hadoop.security.Groups.class)
-).getLogger().setLevel(Level.OFF);
+).getLogger().setLevel(Level.ERROR);
((Log4JLogger)org.apache.hadoop.ipc.Server.LOG
-).getLogger().setLevel(Level.OFF);
+).getLogger().setLevel(Level.ERROR);
((Log4JLogger)org.apache.hadoop.util.AsyncDiskService.LOG
-).getLogger().setLevel(Level.OFF);
+).getLogger().setLevel(Level.ERROR);
}
private static final String inputDir = input;
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java?rev=1392492r1=1392491r2=1392492view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
Mon Oct 1 18:15:44 2012
@@ -61,9 +61,9 @@ import org.junit.Ignore;
public class TestCopyFiles extends TestCase {
{
((Log4JLogger)LogFactory.getLog(org.apache.hadoop.hdfs.StateChange)
-).getLogger().setLevel(Level.OFF);
-((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF);
-
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
+).getLogger().setLevel(Level.ERROR);
+((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ERROR);
+
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ERROR);
((Log4JLogger)DistCpV1.LOG).getLogger().setLevel(Level.ALL);
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java?rev=1392492r1=1392491r2=1392492view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
Mon Oct 1 18:15:44 2012
@@ -46,9 +46,9 @@ import org.junit.Ignore;
public class TestDistCh extends junit.framework.TestCase {
{
((Log4JLogger)LogFactory.getLog(org.apache.hadoop.hdfs.StateChange)
-).getLogger().setLevel(Level.OFF);
-((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF);
-
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
+).getLogger().setLevel(Level.ERROR);
+((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ERROR);
+
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ERROR);
}
static final Long RANDOM_NUMBER_GENERATOR_SEED = null;
svn commit: r1392519 - in /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src: main/java/org/apache/hadoop/tools/OptionsParser.java test/java/org/apache/hadoop/tools/TestOptionsParser.j
Author: bobby
Date: Mon Oct 1 18:59:55 2012
New Revision: 1392519
URL: http://svn.apache.org/viewvc?rev=1392519view=rev
Log:
svn merge -c 1382119 FIXES: HADOOP-8775. MR2 distcp permits non-positive value
to -bandwidth option which causes job never to complete. Contributed by Sandy
Ryza.
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1392519r1=1392518r2=1392519view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
Mon Oct 1 18:59:55 2012
@@ -156,6 +156,10 @@ public class OptionsParser {
try {
Integer mapBandwidth = Integer.parseInt(
getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim());
+if (mapBandwidth.intValue() = 0) {
+ throw new IllegalArgumentException(Bandwidth specified is not
positive: +
+ mapBandwidth);
+}
option.setMapBandwidth(mapBandwidth);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(Bandwidth specified is invalid: +
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1392519r1=1392518r2=1392519view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Mon Oct 1 18:59:55 2012
@@ -110,6 +110,24 @@ public class TestOptionsParser {
hdfs://localhost:8020/target/});
Assert.assertEquals(options.getMapBandwidth(), 11);
}
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testParseNonPositiveBandwidth() {
+OptionsParser.parse(new String[] {
+-bandwidth,
+-11,
+hdfs://localhost:8020/source/first,
+hdfs://localhost:8020/target/});
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testParseZeroBandwidth() {
+OptionsParser.parse(new String[] {
+-bandwidth,
+0,
+hdfs://localhost:8020/source/first,
+hdfs://localhost:8020/target/});
+ }
@Test
public void testParseSkipCRC() {
svn commit: r1392521 - /hadoop/common/branches/branch-0.23/hadoop-project/pom.xml
Author: bobby
Date: Mon Oct 1 19:03:27 2012
New Revision: 1392521
URL: http://svn.apache.org/viewvc?rev=1392521view=rev
Log:
svn merge -c 1392466 FIXES: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError
JVM option in the forked tests. Contributed by Ivan A. Veselovsky.
Modified:
hadoop/common/branches/branch-0.23/hadoop-project/pom.xml
Modified: hadoop/common/branches/branch-0.23/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-project/pom.xml?rev=1392521r1=1392520r2=1392521view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-project/pom.xml Mon Oct 1
19:03:27 2012
@@ -855,7 +855,7 @@
configuration
forkModealways/forkMode
forkedProcessTimeoutInSeconds900/forkedProcessTimeoutInSeconds
- argLine-Xmx1024m/argLine
+ argLine-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError/argLine
environmentVariables
LD_LIBRARY_PATH${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib/LD_LIBRARY_PATH
/environmentVariables
svn commit: r1392521 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Mon Oct 1 19:03:27 2012 New Revision: 1392521 URL: http://svn.apache.org/viewvc?rev=1392521view=rev Log: svn merge -c 1392466 FIXES: HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked tests. Contributed by Ivan A. Veselovsky. Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392521r1=1392520r2=1392521view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Mon Oct 1 19:03:27 2012 @@ -14,6 +14,9 @@ Release 0.23.5 - UNRELEASED HADOOP-8789. Tests setLevel(Level.OFF) should be Level.ERROR. (Andy Isaacson via eli) +HADOOP-8851. Use -XX:+HeapDumpOnOutOfMemoryError JVM option in the forked +tests. (Ivan A. Veselovsky via atm) + OPTIMIZATIONS HADOOP-8819. Incorrectly is used instead of in some file system
svn commit: r1392627 - in /hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java src/test/java/org/apache/had
Author: bobby
Date: Mon Oct 1 21:29:34 2012
New Revision: 1392627
URL: http://svn.apache.org/viewvc?rev=1392627view=rev
Log:
svn merge -c 1392622 FIXES: HADOOP-8310. FileContext#checkPath should handle
URIs with no port. Contributed by Aaron T. Myers.
Added:
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
- copied unchanged from r1392622,
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java
Modified:
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
Modified:
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1392627r1=1392626r2=1392627view=diff
==
---
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/CHANGES.txt
Mon Oct 1 21:29:34 2012
@@ -1,17 +1,5 @@
Hadoop Change Log
-Release 0.23.5 - UNRELEASED
-
- INCOMPATIBLE CHANGES
-
- NEW FEATURES
-
- IMPROVEMENTS
-
- OPTIMIZATIONS
-
- BUG FIXES
-
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -37,6 +25,8 @@ Release 0.23.4 - UNRELEASED
HADOOP-8623. hadoop jar command should respect HADOOP_OPTS.
(Steven Willis via suresh)
+HADOOP-8310. FileContext#checkPath should handle URIs with no port. (atm)
+
Release 0.23.3
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1392627r1=1392626r2=1392627view=diff
==
---
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
(original)
+++
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
Mon Oct 1 21:29:34 2012
@@ -352,20 +352,23 @@ public abstract class AbstractFileSystem
}
}
String thisScheme = this.getUri().getScheme();
-String thisAuthority = this.getUri().getAuthority();
+String thisHost = this.getUri().getHost();
+String thatHost = uri.getHost();
-// Schemes and authorities must match.
+// Schemes and hosts must match.
// Allow for null Authority for file:///
if (!thisScheme.equalsIgnoreCase(thatScheme) ||
- (thisAuthority != null
-!thisAuthority.equalsIgnoreCase(thatAuthority)) ||
- (thisAuthority == null thatAuthority != null)) {
+ (thisHost != null
+!thisHost.equalsIgnoreCase(thatHost)) ||
+ (thisHost == null thatHost != null)) {
throw new InvalidPathException(Wrong FS: + path + , expected:
+ this.getUri());
}
+// Ports must match, unless this FS instance is using the default port, in
+// which case the port may be omitted from the given URI
int thisPort = this.getUri().getPort();
-int thatPort = path.toUri().getPort();
+int thatPort = uri.getPort();
if (thatPort == -1) { // -1 = defaultPort of Uri scheme
thatPort = this.getUriDefaultPort();
}
svn commit: r1392631 - /hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: bobby
Date: Mon Oct 1 21:31:24 2012
New Revision: 1392631
URL: http://svn.apache.org/viewvc?rev=1392631view=rev
Log:
Updated release notes for 0.23.4-rc1
Modified:
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Modified:
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html?rev=1392631r1=1392630r2=1392631view=diff
==
---
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
(original)
+++
hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Mon Oct 1 21:31:24 2012
@@ -12,6 +12,16 @@ These release notes include new develope
a name=changes/
h2Changes since Hadoop 0.23.3/h2
ul
+li a href=https://issues.apache.org/jira/browse/YARN-137;YARN-137/a.
+ Major improvement reported by Siddharth Seth and fixed by Siddharth Seth
(scheduler)br
+ bChange the default scheduler to the CapacityScheduler/bbr
+ blockquoteThere's some bugs in the FifoScheduler atm - doesn't
distribute tasks across nodes and some headroom (available resource) issues.
+That's not the best experience for users trying out the 2.0 branch. The CS
with the default configuration of a single queue behaves the same as the
FifoScheduler and doesn't have these issues.
+/blockquote/li
+li a href=https://issues.apache.org/jira/browse/YARN-108;YARN-108/a.
+ Critical bug reported by Jason Lowe and fixed by Jason Lowe
(nodemanager)br
+ bFSDownload can create cache directories with the wrong
permissions/bbr
+ blockquoteWhen the cluster is configured with a restrictive umask,
e.g.: {{fs.permissions.umask-mode=0077}}, the nodemanager can end up creating
directory entries in the public cache with the wrong permissions. The
permissions can end up where only the nodemanager user can access files in the
public cache, preventing jobs from running properly./blockquote/li
li a href=https://issues.apache.org/jira/browse/YARN-106;YARN-106/a.
Major bug reported by Jason Lowe and fixed by Jason Lowe (nodemanager)br
bNodemanager needs to set permissions of local directories/bbr
@@ -144,6 +154,10 @@ java.lang.NullPointerException
Minor improvement reported by Steven Willis and fixed by Steven Willis
(scripts)br
bhadoop jar command should respect HADOOP_OPTS/bbr
blockquote/blockquote/li
+li a
href=https://issues.apache.org/jira/browse/HADOOP-8310;HADOOP-8310/a.
+ Major bug reported by Aaron T. Myers and fixed by Aaron T. Myers (fs)br
+ bFileContext#checkPath should handle URIs with no port/bbr
+ blockquote/blockquote/li
li a
href=https://issues.apache.org/jira/browse/HADOOP-8183;HADOOP-8183/a.
Minor improvement reported by Harsh J and fixed by Harsh J (util)br
bStop using mapred.used.genericoptionsparser to avoid unnecessary
warnings/bbr
svn commit: r1392671 - /hadoop/common/tags/release-0.23.4-rc1/
Author: bobby Date: Mon Oct 1 22:38:52 2012 New Revision: 1392671 URL: http://svn.apache.org/viewvc?rev=1392671view=rev Log: Hadoop 0.23.4-rc1 release. Added: hadoop/common/tags/release-0.23.4-rc1/ (props changed) - copied from r1392658, hadoop/common/branches/branch-0.23.4/ Propchange: hadoop/common/tags/release-0.23.4-rc1/ -- --- svn:ignore (added) +++ svn:ignore Mon Oct 1 22:38:52 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/tags/release-0.23.4-rc1/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Oct 1 22:38:52 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1391805 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Sat Sep 29 14:15:39 2012 New Revision: 1391805 URL: http://svn.apache.org/viewvc?rev=1391805view=rev Log: Preparing for release 0.23.4 Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1391805r1=1391804r2=1391805view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Sat Sep 29 14:15:39 2012 @@ -993,6 +993,18 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via bobby) +Release 0.23.5 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES @@ -1010,7 +1022,7 @@ Release 0.23.4 - UNRELEASED HADOOP-8843. Old trash directories are never deleted on upgrade from 1.x (jlowe) -Release 0.23.3 - UNRELEASED +Release 0.23.3 INCOMPATIBLE CHANGES
svn commit: r1391806 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Sat Sep 29 14:18:27 2012 New Revision: 1391806 URL: http://svn.apache.org/viewvc?rev=1391806view=rev Log: Preparing for release 0.23.4 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1391806r1=1391805r2=1391806view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Sat Sep 29 14:18:27 2012 @@ -739,6 +739,18 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via bobby) +Release 0.23.5 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES @@ -759,7 +771,7 @@ Release 0.23.4 - UNRELEASED HADOOP-8684. Deadlock between WritableComparator and WritableComparable. (Jing Zhao via suresh) -Release 0.23.3 - UNRELEASED +Release 0.23.3 INCOMPATIBLE CHANGES
svn commit: r1391807 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Sat Sep 29 14:24:08 2012 New Revision: 1391807 URL: http://svn.apache.org/viewvc?rev=1391807view=rev Log: Preparing for release 0.23.4 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1391807r1=1391806r2=1391807view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Sat Sep 29 14:24:08 2012 @@ -1,5 +1,17 @@ Hadoop Change Log +Release 0.23.5 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES @@ -25,7 +37,7 @@ Release 0.23.4 - UNRELEASED HADOOP-8623. hadoop jar command should respect HADOOP_OPTS. (Steven Willis via suresh) -Release 0.23.3 - UNRELEASED +Release 0.23.3 INCOMPATIBLE CHANGES
svn commit: r1391808 - /hadoop/common/branches/branch-0.23.4/
Author: bobby Date: Sat Sep 29 14:25:59 2012 New Revision: 1391808 URL: http://svn.apache.org/viewvc?rev=1391808view=rev Log: Creating temporary branch for 0.23.4 release Added: hadoop/common/branches/branch-0.23.4/ (props changed) - copied from r1391807, hadoop/common/branches/branch-0.23/ Propchange: hadoop/common/branches/branch-0.23.4/ -- --- svn:ignore (added) +++ svn:ignore Sat Sep 29 14:25:59 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/branch-0.23.4/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Sat Sep 29 14:25:59 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1391812 - in /hadoop/common/branches/branch-0.23/hadoop-common-project: hadoop-annotations/pom.xml hadoop-auth-examples/pom.xml hadoop-auth/pom.xml hadoop-common/pom.xml pom.xml
Author: bobby Date: Sat Sep 29 14:32:47 2012 New Revision: 1391812 URL: http://svn.apache.org/viewvc?rev=1391812view=rev Log: Preparing for release 0.23.4 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-annotations/pom.xml hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth-examples/pom.xml hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth/pom.xml hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml hadoop/common/branches/branch-0.23/hadoop-common-project/pom.xml Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-annotations/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-annotations/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-annotations/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-annotations/pom.xml Sat Sep 29 14:32:47 2012 @@ -21,12 +21,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-annotations/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version descriptionApache Hadoop Annotations/description nameApache Hadoop Annotations/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth-examples/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth-examples/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth-examples/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth-examples/pom.xml Sat Sep 29 14:32:47 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-auth-examples/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version packagingwar/packaging nameApache Hadoop Auth Examples/name Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-auth/pom.xml Sat Sep 29 14:32:47 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-auth/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version packagingjar/packaging nameApache Hadoop Auth/name Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/pom.xml Sat Sep 29 14:32:47 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project-dist/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../../hadoop-project-dist/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-common/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version descriptionApache Hadoop Common/description nameApache Hadoop Common/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop
svn commit: r1391812 - in /hadoop/common/branches/branch-0.23: ./ hadoop-assemblies/ hadoop-client/ hadoop-dist/ hadoop-minicluster/ hadoop-project-dist/ hadoop-project/ hadoop-tools/ hadoop-tools/had
Author: bobby Date: Sat Sep 29 14:32:47 2012 New Revision: 1391812 URL: http://svn.apache.org/viewvc?rev=1391812view=rev Log: Preparing for release 0.23.4 Modified: hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml hadoop/common/branches/branch-0.23/hadoop-client/pom.xml hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml hadoop/common/branches/branch-0.23/hadoop-project-dist/pom.xml hadoop/common/branches/branch-0.23/hadoop-project/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-archives/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-datajoin/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-extras/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-gridmix/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-pipes/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-rumen/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-tools-dist/pom.xml hadoop/common/branches/branch-0.23/hadoop-tools/pom.xml hadoop/common/branches/branch-0.23/pom.xml Modified: hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-assemblies/pom.xml Sat Sep 29 14:32:47 2012 @@ -23,12 +23,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-assemblies/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version nameApache Hadoop Assemblies/name descriptionApache Hadoop Assemblies/description Modified: hadoop/common/branches/branch-0.23/hadoop-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-client/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-client/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-client/pom.xml Sat Sep 29 14:32:47 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-client/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version packagingjar/packaging descriptionApache Hadoop Client/description Modified: hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-dist/pom.xml Sat Sep 29 14:32:47 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-dist/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version descriptionApache Hadoop Distribution/description nameApache Hadoop Distribution/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml?rev=1391812r1=1391811r2=1391812view=diff == --- hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-minicluster/pom.xml Sat Sep 29 14:32:47 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.5-SNAPSHOT/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-minicluster/artifactId - version0.23.4-SNAPSHOT/version + version0.23.5-SNAPSHOT/version packagingjar/packaging descriptionApache Hadoop Mini
svn commit: r1391813 - in /hadoop/common/branches/branch-0.23.4/hadoop-common-project: hadoop-annotations/pom.xml hadoop-auth-examples/pom.xml hadoop-auth/pom.xml hadoop-common/pom.xml pom.xml
Author: bobby Date: Sat Sep 29 14:37:45 2012 New Revision: 1391813 URL: http://svn.apache.org/viewvc?rev=1391813view=rev Log: Updated version numbers for 0.23.4 release Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-annotations/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth-examples/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-common-project/pom.xml Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-annotations/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-annotations/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-annotations/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-annotations/pom.xml Sat Sep 29 14:37:45 2012 @@ -21,12 +21,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-annotations/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version descriptionApache Hadoop Annotations/description nameApache Hadoop Annotations/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth-examples/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth-examples/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth-examples/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth-examples/pom.xml Sat Sep 29 14:37:45 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-auth-examples/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version packagingwar/packaging nameApache Hadoop Auth Examples/name Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-auth/pom.xml Sat Sep 29 14:37:45 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-auth/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version packagingjar/packaging nameApache Hadoop Auth/name Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/pom.xml Sat Sep 29 14:37:45 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project-dist/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../../hadoop-project-dist/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-common/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version descriptionApache Hadoop Common/description nameApache Hadoop Common/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-common-project/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-common
svn commit: r1391813 - in /hadoop/common/branches/branch-0.23.4: ./ hadoop-assemblies/ hadoop-client/ hadoop-dist/ hadoop-minicluster/ hadoop-project-dist/ hadoop-project/ hadoop-tools/ hadoop-tools/h
Author: bobby Date: Sat Sep 29 14:37:45 2012 New Revision: 1391813 URL: http://svn.apache.org/viewvc?rev=1391813view=rev Log: Updated version numbers for 0.23.4 release Modified: hadoop/common/branches/branch-0.23.4/hadoop-assemblies/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-client/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-dist/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-minicluster/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-project-dist/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-project/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-archives/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-datajoin/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-distcp/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-extras/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-gridmix/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-pipes/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-rumen/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-streaming/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/hadoop-tools-dist/pom.xml hadoop/common/branches/branch-0.23.4/hadoop-tools/pom.xml hadoop/common/branches/branch-0.23.4/pom.xml Modified: hadoop/common/branches/branch-0.23.4/hadoop-assemblies/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-assemblies/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-assemblies/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-assemblies/pom.xml Sat Sep 29 14:37:45 2012 @@ -23,12 +23,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-assemblies/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version nameApache Hadoop Assemblies/name descriptionApache Hadoop Assemblies/description Modified: hadoop/common/branches/branch-0.23.4/hadoop-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-client/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-client/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-client/pom.xml Sat Sep 29 14:37:45 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-client/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version packagingjar/packaging descriptionApache Hadoop Client/description Modified: hadoop/common/branches/branch-0.23.4/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-dist/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-dist/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-dist/pom.xml Sat Sep 29 14:37:45 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-dist/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version descriptionApache Hadoop Distribution/description nameApache Hadoop Distribution/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.4/hadoop-minicluster/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.4/hadoop-minicluster/pom.xml?rev=1391813r1=1391812r2=1391813view=diff == --- hadoop/common/branches/branch-0.23.4/hadoop-minicluster/pom.xml (original) +++ hadoop/common/branches/branch-0.23.4/hadoop-minicluster/pom.xml Sat Sep 29 14:37:45 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.4-SNAPSHOT/version +version0.23.4/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-minicluster/artifactId - version0.23.4-SNAPSHOT/version + version0.23.4/version packagingjar/packaging descriptionApache Hadoop
svn commit: r1391814 [1/2] - /hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: bobby Date: Sat Sep 29 14:39:24 2012 New Revision: 1391814 URL: http://svn.apache.org/viewvc?rev=1391814view=rev Log: Updated release notes for 0.23.4 release Modified: hadoop/common/branches/branch-0.23.4/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
svn commit: r1391816 - /hadoop/common/tags/release-0.23.4-rc0/
Author: bobby Date: Sat Sep 29 14:41:10 2012 New Revision: 1391816 URL: http://svn.apache.org/viewvc?rev=1391816view=rev Log: Hadoop 0.23.4-rc0 release. Added: hadoop/common/tags/release-0.23.4-rc0/ (props changed) - copied from r1391815, hadoop/common/branches/branch-0.23.4/ Propchange: hadoop/common/tags/release-0.23.4-rc0/ -- --- svn:ignore (added) +++ svn:ignore Sat Sep 29 14:41:10 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/tags/release-0.23.4-rc0/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Sat Sep 29 14:41:10 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1390680 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/io/WritableComparator.java
Author: bobby
Date: Wed Sep 26 19:10:40 2012
New Revision: 1390680
URL: http://svn.apache.org/viewvc?rev=1390680view=rev
Log:
svn merge -c 1379506 FIXES: HADOOP-8684. Deadlock between WritableComparator
and WritableComparable. Contributed by Jing Zhao
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1390680r1=1390679r2=1390680view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Sep 26 19:10:40 2012
@@ -750,6 +750,9 @@ Release 0.23.4 - UNRELEASED
HADOOP-8843. Old trash directories are never deleted on upgrade
from 1.x (jlowe)
+HADOOP-8684. Deadlock between WritableComparator and WritableComparable.
+(Jing Zhao via suresh)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java?rev=1390680r1=1390679r2=1390680view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableComparator.java
Wed Sep 26 19:10:40 2012
@@ -18,8 +18,9 @@
package org.apache.hadoop.io;
-import java.io.*;
-import java.util.*;
+import java.io.DataInput;
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -38,12 +39,11 @@ import org.apache.hadoop.util.Reflection
@InterfaceStability.Stable
public class WritableComparator implements RawComparator {
- private static HashMapClass, WritableComparator comparators =
-new HashMapClass, WritableComparator(); // registry
+ private static final ConcurrentHashMapClass, WritableComparator
comparators
+ = new ConcurrentHashMapClass, WritableComparator(); // registry
/** Get a comparator for a {@link WritableComparable} implementation. */
- public static synchronized
- WritableComparator get(Class? extends WritableComparable c) {
+ public static WritableComparator get(Class? extends WritableComparable c) {
WritableComparator comparator = comparators.get(c);
if (comparator == null) {
// force the static initializers to run
@@ -76,12 +76,10 @@ public class WritableComparator implemen
/** Register an optimized comparator for a {@link WritableComparable}
* implementation. Comparators registered with this method must be
* thread-safe. */
- public static synchronized void define(Class c,
- WritableComparator comparator) {
+ public static void define(Class c, WritableComparator comparator) {
comparators.put(c, comparator);
}
-
private final Class? extends WritableComparable keyClass;
private final WritableComparable key1;
private final WritableComparable key2;
svn commit: r1389875 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: bobby Date: Tue Sep 25 14:22:17 2012 New Revision: 1389875 URL: http://svn.apache.org/viewvc?rev=1389875view=rev Log: HADOOP-8838. Colorize the test-patch output sent to JIRA (Harsh J via bobby) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1389875r1=1389874r2=1389875view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Sep 25 14:22:17 2012 @@ -111,6 +111,9 @@ Trunk (Unreleased) HADOOP-7930. Kerberos relogin interval in UserGroupInformation should be configurable (Robert Kanter via harsh) +HADOOP-8838. Colorize the test-patch output sent to JIRA (Harsh J via +bobby) + BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName.
