hadoop git commit: HADOOP-12082. Support multiple authentication schemes via AuthenticationFilter. Contributed by Hrishikesh Gadre.
Repository: hadoop Updated Branches: refs/heads/branch-2 6bb23a14b -> b913b62aa HADOOP-12082. Support multiple authentication schemes via AuthenticationFilter. Contributed by Hrishikesh Gadre. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b913b62a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b913b62a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b913b62a Branch: refs/heads/branch-2 Commit: b913b62aafed8ac5638e858fa51d5943ce757196 Parents: 6bb23a1 Author: Benoy Antony Authored: Tue Oct 25 08:30:02 2016 -0700 Committer: Benoy Antony Committed: Tue Oct 25 08:30:02 2016 -0700 -- hadoop-common-project/hadoop-auth/pom.xml | 24 ++ .../client/KerberosAuthenticator.java | 8 +- .../server/AuthenticationFilter.java| 47 ++- .../server/AuthenticationHandler.java | 2 +- .../server/AuthenticationHandlerUtil.java | 105 ++ .../server/CompositeAuthenticationHandler.java | 30 ++ .../authentication/server/HttpConstants.java| 55 +++ .../server/LdapAuthenticationHandler.java | 339 +++ .../MultiSchemeAuthenticationHandler.java | 209 .../authentication/server/package-info.java | 27 ++ .../src/site/markdown/Configuration.md | 137 .../client/TestKerberosAuthenticator.java | 71 +++- .../authentication/server/LdapConstants.java| 31 ++ .../server/TestLdapAuthenticationHandler.java | 159 + .../TestMultiSchemeAuthenticationHandler.java | 189 +++ .../DelegationTokenAuthenticationFilter.java| 9 +- .../DelegationTokenAuthenticationHandler.java | 25 +- ...emeDelegationTokenAuthenticationHandler.java | 182 ++ hadoop-project/pom.xml | 3 + 19 files changed, 1630 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b913b62a/hadoop-common-project/hadoop-auth/pom.xml -- diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index e138382..1f702b2 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -154,6 +154,30 @@ curator-test test + + org.apache.directory.server + apacheds-server-integ + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-schema-data + + + + + org.apache.directory.server + apacheds-core-integ + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-schema-data + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b913b62a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java index a69ee46..ceec927 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java @@ -14,6 +14,7 @@ package org.apache.hadoop.security.authentication.client; import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.security.authentication.server.HttpConstants; import org.apache.hadoop.security.authentication.util.AuthToken; import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.ietf.jgss.GSSContext; @@ -57,17 +58,18 @@ public class KerberosAuthenticator implements Authenticator { /** * HTTP header used by the SPNEGO server endpoint during an authentication sequence. */ - public static final String WWW_AUTHENTICATE = "WWW-Authenticate"; + public static final String WWW_AUTHENTICATE = + HttpConstants.WWW_AUTHENTICATE_HEADER; /** * HTTP header used by the SPNEGO client endpoint during an authentication sequence. */ - public static final String AUTHORIZATION = "Authorization"; + public static final String AUTHORIZATION = HttpConstants.AUTHORIZATION_HEADER; /** * HTTP header prefix used by the SPNEGO client/server endpoints during an authentication sequence. */ - public static final String NEGOTIATE = "Negotiate"; + public static final String NEGOTIATE = Http
hadoop git commit: HADOOP-12082 Support multiple authentication schemes via AuthenticationFilter. Contributed by Hrishikesh Gadre.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 b82364759 -> 653ceab7d HADOOP-12082 Support multiple authentication schemes via AuthenticationFilter. Contributed by Hrishikesh Gadre. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/653ceab7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/653ceab7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/653ceab7 Branch: refs/heads/branch-2.8 Commit: 653ceab7d35670d488e1dfc7500ca281bc4327e0 Parents: b823647 Author: Benoy Antony Authored: Fri Oct 21 19:43:22 2016 -0700 Committer: Benoy Antony Committed: Fri Oct 21 19:43:22 2016 -0700 -- hadoop-common-project/hadoop-auth/pom.xml | 24 ++ .../client/KerberosAuthenticator.java | 8 +- .../server/AuthenticationFilter.java| 47 ++- .../server/AuthenticationHandler.java | 2 +- .../server/AuthenticationHandlerUtil.java | 105 ++ .../server/CompositeAuthenticationHandler.java | 30 ++ .../authentication/server/HttpConstants.java| 55 +++ .../server/LdapAuthenticationHandler.java | 339 +++ .../MultiSchemeAuthenticationHandler.java | 209 .../authentication/server/package-info.java | 27 ++ .../src/site/markdown/Configuration.md | 137 .../client/TestKerberosAuthenticator.java | 71 +++- .../authentication/server/LdapConstants.java| 31 ++ .../server/TestLdapAuthenticationHandler.java | 159 + .../TestMultiSchemeAuthenticationHandler.java | 189 +++ .../DelegationTokenAuthenticationFilter.java| 9 +- .../DelegationTokenAuthenticationHandler.java | 25 +- ...emeDelegationTokenAuthenticationHandler.java | 182 ++ hadoop-project/pom.xml | 3 + 19 files changed, 1630 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/653ceab7/hadoop-common-project/hadoop-auth/pom.xml -- diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index f59cf84..9d99a05 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -154,6 +154,30 @@ curator-test test + + org.apache.directory.server + apacheds-server-integ + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-schema-data + + + + + org.apache.directory.server + apacheds-core-integ + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-schema-data + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/653ceab7/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java index a69ee46..ceec927 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java @@ -14,6 +14,7 @@ package org.apache.hadoop.security.authentication.client; import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.security.authentication.server.HttpConstants; import org.apache.hadoop.security.authentication.util.AuthToken; import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.ietf.jgss.GSSContext; @@ -57,17 +58,18 @@ public class KerberosAuthenticator implements Authenticator { /** * HTTP header used by the SPNEGO server endpoint during an authentication sequence. */ - public static final String WWW_AUTHENTICATE = "WWW-Authenticate"; + public static final String WWW_AUTHENTICATE = + HttpConstants.WWW_AUTHENTICATE_HEADER; /** * HTTP header used by the SPNEGO client endpoint during an authentication sequence. */ - public static final String AUTHORIZATION = "Authorization"; + public static final String AUTHORIZATION = HttpConstants.AUTHORIZATION_HEADER; /** * HTTP header prefix used by the SPNEGO client/server endpoints during an authentication sequence. */ - public static final String NEGOTIATE = "Negotiate"; + public static final String NEGOTIATE = Http
hadoop git commit: HADOOP-12082 Support multiple authentication schemes via AuthenticationFilter
Repository: hadoop Updated Branches: refs/heads/trunk d75cbc574 -> 4bca38524 HADOOP-12082 Support multiple authentication schemes via AuthenticationFilter Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bca3852 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bca3852 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bca3852 Branch: refs/heads/trunk Commit: 4bca385241c0fc8ff168c7b0f2984a7aed2c7492 Parents: d75cbc5 Author: Benoy Antony Authored: Tue Oct 18 18:32:01 2016 -0700 Committer: Benoy Antony Committed: Tue Oct 18 18:32:01 2016 -0700 -- hadoop-common-project/hadoop-auth/pom.xml | 42 +++ .../client/KerberosAuthenticator.java | 8 +- .../server/AuthenticationFilter.java| 47 ++- .../server/AuthenticationHandler.java | 2 +- .../server/AuthenticationHandlerUtil.java | 105 ++ .../server/CompositeAuthenticationHandler.java | 30 ++ .../authentication/server/HttpConstants.java| 55 +++ .../server/LdapAuthenticationHandler.java | 339 +++ .../MultiSchemeAuthenticationHandler.java | 209 .../authentication/server/package-info.java | 27 ++ .../src/site/markdown/Configuration.md | 137 .../client/TestKerberosAuthenticator.java | 71 +++- .../authentication/server/LdapConstants.java| 31 ++ .../server/TestLdapAuthenticationHandler.java | 159 + .../TestMultiSchemeAuthenticationHandler.java | 189 +++ .../DelegationTokenAuthenticationFilter.java| 9 +- .../DelegationTokenAuthenticationHandler.java | 25 +- ...emeDelegationTokenAuthenticationHandler.java | 182 ++ hadoop-project/pom.xml | 4 + 19 files changed, 1649 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bca3852/hadoop-common-project/hadoop-auth/pom.xml -- diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 4cbdc49..0b37715 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -135,6 +135,48 @@ org.apache.kerby kerb-simplekdc + + org.apache.directory.server + apacheds-core + ${apacheds.version} + test + + + org.apache.directory.server + apacheds-protocol-ldap + ${apacheds.version} + test + + + org.apache.directory.server + apacheds-ldif-partition + ${apacheds.version} + test + + + org.apache.directory.api + api-ldap-codec-core + ${ldap-api.version} + test + + + org.apache.directory.api + api-ldap-model + ${ldap-api.version} + test + + + org.apache.directory.server + apacheds-server-integ + ${apacheds.version} + test + + + org.apache.directory.server + apacheds-core-integ + ${apacheds.version} + test + http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bca3852/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java index a69ee46..ceec927 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java @@ -14,6 +14,7 @@ package org.apache.hadoop.security.authentication.client; import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.security.authentication.server.HttpConstants; import org.apache.hadoop.security.authentication.util.AuthToken; import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.ietf.jgss.GSSContext; @@ -57,17 +58,18 @@ public class KerberosAuthenticator implements Authenticator { /** * HTTP header used by the SPNEGO server endpoint during an authentication sequence. */ - public static final String WWW_AUTHENTICATE = "WWW-Authenticate"; + public static final String WWW_AUTHENTICATE = + HttpConstants.WWW_AUTHENTICATE_HEADER; /** * HTTP header used by the SPNEGO client endpoint during an authentication sequence. */ - public static final String AUTHORIZATION = "Authorization&
hadoop git commit: HDFS-10735 Distcp using webhdfs on secure HA clusters fails with StandbyException
Repository: hadoop Updated Branches: refs/heads/branch-2.8 8c71fc122 -> ca20ffea4 HDFS-10735 Distcp using webhdfs on secure HA clusters fails with StandbyException (cherry picked from commit 701c27a7762294e1a5fb2b3ac81f5534aa37f667) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca20ffea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca20ffea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca20ffea Branch: refs/heads/branch-2.8 Commit: ca20ffea462de7529f8ce2a36f85fae56f814cbd Parents: 8c71fc1 Author: Benoy Antony Authored: Fri Oct 14 10:26:39 2016 -0700 Committer: Benoy Antony Committed: Fri Oct 14 10:32:37 2016 -0700 -- .../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 8 1 file changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca20ffea/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index aeac1a3..d696081 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -89,6 +89,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryUtils; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; @@ -475,6 +476,13 @@ public class WebHdfsFileSystem extends FileSystem } IOException re = JsonUtilClient.toRemoteException(m); + + //check if exception is due to communication with a Standby name node + if (re.getMessage() != null && re.getMessage().endsWith( + StandbyException.class.getSimpleName())) { +LOG.trace("Detected StandbyException", re); +throw new IOException(re); + } // extract UGI-related exceptions and unwrap InvalidToken // the NN mangles these exceptions but the DN does not and may need // to re-fetch a token if either report the token is expired - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-10735 Distcp using webhdfs on secure HA clusters fails with StandbyException
Repository: hadoop Updated Branches: refs/heads/branch-2 6e0b55fe7 -> 156b92e36 HDFS-10735 Distcp using webhdfs on secure HA clusters fails with StandbyException (cherry picked from commit 701c27a7762294e1a5fb2b3ac81f5534aa37f667) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/156b92e3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/156b92e3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/156b92e3 Branch: refs/heads/branch-2 Commit: 156b92e36bb04252a1c3ee7b334809021d963254 Parents: 6e0b55f Author: Benoy Antony Authored: Fri Oct 14 10:26:39 2016 -0700 Committer: Benoy Antony Committed: Fri Oct 14 10:29:42 2016 -0700 -- .../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 8 1 file changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/156b92e3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 92e6901..7f9a02e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -90,6 +90,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryUtils; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; @@ -479,6 +480,13 @@ public class WebHdfsFileSystem extends FileSystem } IOException re = JsonUtilClient.toRemoteException(m); + + //check if exception is due to communication with a Standby name node + if (re.getMessage() != null && re.getMessage().endsWith( + StandbyException.class.getSimpleName())) { +LOG.trace("Detected StandbyException", re); +throw new IOException(re); + } // extract UGI-related exceptions and unwrap InvalidToken // the NN mangles these exceptions but the DN does not and may need // to re-fetch a token if either report the token is expired - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-10735 Distcp using webhdfs on secure HA clusters fails with StandbyException
Repository: hadoop Updated Branches: refs/heads/trunk 8a9f6635a -> 701c27a77 HDFS-10735 Distcp using webhdfs on secure HA clusters fails with StandbyException Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/701c27a7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/701c27a7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/701c27a7 Branch: refs/heads/trunk Commit: 701c27a7762294e1a5fb2b3ac81f5534aa37f667 Parents: 8a9f663 Author: Benoy Antony Authored: Fri Oct 14 10:26:39 2016 -0700 Committer: Benoy Antony Committed: Fri Oct 14 10:26:39 2016 -0700 -- .../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 8 1 file changed, 8 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/701c27a7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 19de5b5..af43d56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -91,6 +91,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryUtils; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; @@ -471,6 +472,13 @@ public class WebHdfsFileSystem extends FileSystem } IOException re = JsonUtilClient.toRemoteException(m); + + //check if exception is due to communication with a Standby name node + if (re.getMessage() != null && re.getMessage().endsWith( + StandbyException.class.getSimpleName())) { +LOG.trace("Detected StandbyException", re); +throw new IOException(re); + } // extract UGI-related exceptions and unwrap InvalidToken // the NN mangles these exceptions but the DN does not and may need // to re-fetch a token if either report the token is expired - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-12929. JWTRedirectAuthenticationHandler must accommodate null expiration time. Contributed by Larry McCay.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 29872506b -> 1b263ea49 HADOOP-12929. JWTRedirectAuthenticationHandler must accommodate null expiration time. Contributed by Larry McCay. (cherry picked from commit e7ed05e4f5b0421e93f2f2cadc5beda3d28b9911) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b263ea4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b263ea4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b263ea4 Branch: refs/heads/branch-2.8 Commit: 1b263ea49056a370e2308f0f88bbb68d538e4c03 Parents: 2987250 Author: Benoy Antony Authored: Mon Mar 21 13:19:43 2016 -0700 Committer: Benoy Antony Committed: Mon Mar 21 13:31:55 2016 -0700 -- .../JWTRedirectAuthenticationHandler.java | 16 ++--- .../TestJWTRedirectAuthentictionHandler.java| 37 2 files changed, 33 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b263ea4/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java index cbe923b..61f5b9e 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java @@ -26,20 +26,10 @@ import java.util.List; import java.util.Properties; import java.text.ParseException; -import java.io.ByteArrayInputStream; -import java.io.UnsupportedEncodingException; -import java.security.PublicKey; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; -import java.security.cert.CertificateException; import java.security.interfaces.RSAPublicKey; -import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.AltKerberosAuthenticationHandler; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; import org.apache.hadoop.security.authentication.util.CertificateUtil; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +73,8 @@ public class JWTRedirectAuthenticationHandler extends private static Logger LOG = LoggerFactory .getLogger(JWTRedirectAuthenticationHandler.class); - public static final String AUTHENTICATION_PROVIDER_URL = "authentication.provider.url"; + public static final String AUTHENTICATION_PROVIDER_URL = + "authentication.provider.url"; public static final String PUBLIC_KEY_PEM = "public.key.pem"; public static final String EXPECTED_JWT_AUDIENCES = "expected.jwt.audiences"; public static final String JWT_COOKIE_NAME = "jwt.cookie.name"; @@ -205,7 +196,6 @@ public class JWTRedirectAuthenticationHandler extends protected String getJWTFromCookie(HttpServletRequest req) { String serializedJWT = null; Cookie[] cookies = req.getCookies(); -String userName = null; if (cookies != null) { for (Cookie cookie : cookies) { if (cookieName.equals(cookie.getName())) { @@ -350,7 +340,7 @@ public class JWTRedirectAuthenticationHandler extends boolean valid = false; try { Date expires = jwtToken.getJWTClaimsSet().getExpirationTime(); - if (expires != null && new Date().before(expires)) { + if (expires == null || new Date().before(expires)) { LOG.debug("JWT token expiration date has been " + "successfully validated"); valid = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b263ea4/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java -- diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java index 019ecb4..97a8a9d 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hado
hadoop git commit: HADOOP-12929. JWTRedirectAuthenticationHandler must accommodate null expiration time. Contributed by Larry McCay.
Repository: hadoop Updated Branches: refs/heads/branch-2 2ce24bf21 -> 4dd02f948 HADOOP-12929. JWTRedirectAuthenticationHandler must accommodate null expiration time. Contributed by Larry McCay. (cherry picked from commit e7ed05e4f5b0421e93f2f2cadc5beda3d28b9911) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4dd02f94 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4dd02f94 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4dd02f94 Branch: refs/heads/branch-2 Commit: 4dd02f948482a4e329bfc6c8baca33517d931121 Parents: 2ce24bf Author: Benoy Antony Authored: Mon Mar 21 13:19:43 2016 -0700 Committer: Benoy Antony Committed: Mon Mar 21 13:21:54 2016 -0700 -- .../JWTRedirectAuthenticationHandler.java | 16 ++--- .../TestJWTRedirectAuthentictionHandler.java| 37 2 files changed, 33 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dd02f94/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java index cbe923b..61f5b9e 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java @@ -26,20 +26,10 @@ import java.util.List; import java.util.Properties; import java.text.ParseException; -import java.io.ByteArrayInputStream; -import java.io.UnsupportedEncodingException; -import java.security.PublicKey; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; -import java.security.cert.CertificateException; import java.security.interfaces.RSAPublicKey; -import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.AltKerberosAuthenticationHandler; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; import org.apache.hadoop.security.authentication.util.CertificateUtil; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +73,8 @@ public class JWTRedirectAuthenticationHandler extends private static Logger LOG = LoggerFactory .getLogger(JWTRedirectAuthenticationHandler.class); - public static final String AUTHENTICATION_PROVIDER_URL = "authentication.provider.url"; + public static final String AUTHENTICATION_PROVIDER_URL = + "authentication.provider.url"; public static final String PUBLIC_KEY_PEM = "public.key.pem"; public static final String EXPECTED_JWT_AUDIENCES = "expected.jwt.audiences"; public static final String JWT_COOKIE_NAME = "jwt.cookie.name"; @@ -205,7 +196,6 @@ public class JWTRedirectAuthenticationHandler extends protected String getJWTFromCookie(HttpServletRequest req) { String serializedJWT = null; Cookie[] cookies = req.getCookies(); -String userName = null; if (cookies != null) { for (Cookie cookie : cookies) { if (cookieName.equals(cookie.getName())) { @@ -350,7 +340,7 @@ public class JWTRedirectAuthenticationHandler extends boolean valid = false; try { Date expires = jwtToken.getJWTClaimsSet().getExpirationTime(); - if (expires != null && new Date().before(expires)) { + if (expires == null || new Date().before(expires)) { LOG.debug("JWT token expiration date has been " + "successfully validated"); valid = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dd02f94/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java -- diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java index 019ecb4..97a8a9d 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hado
hadoop git commit: HADOOP-12929. JWTRedirectAuthenticationHandler must accommodate null expiration time. Contributed by Larry McCay.
Repository: hadoop Updated Branches: refs/heads/trunk 680716f31 -> e7ed05e4f HADOOP-12929. JWTRedirectAuthenticationHandler must accommodate null expiration time. Contributed by Larry McCay. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7ed05e4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7ed05e4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7ed05e4 Branch: refs/heads/trunk Commit: e7ed05e4f5b0421e93f2f2cadc5beda3d28b9911 Parents: 680716f Author: Benoy Antony Authored: Mon Mar 21 13:19:43 2016 -0700 Committer: Benoy Antony Committed: Mon Mar 21 13:19:43 2016 -0700 -- .../JWTRedirectAuthenticationHandler.java | 16 ++--- .../TestJWTRedirectAuthentictionHandler.java| 37 2 files changed, 33 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7ed05e4/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java index cbe923b..61f5b9e 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/JWTRedirectAuthenticationHandler.java @@ -26,20 +26,10 @@ import java.util.List; import java.util.Properties; import java.text.ParseException; -import java.io.ByteArrayInputStream; -import java.io.UnsupportedEncodingException; -import java.security.PublicKey; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; -import java.security.cert.CertificateException; import java.security.interfaces.RSAPublicKey; -import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.AltKerberosAuthenticationHandler; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; import org.apache.hadoop.security.authentication.util.CertificateUtil; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +73,8 @@ public class JWTRedirectAuthenticationHandler extends private static Logger LOG = LoggerFactory .getLogger(JWTRedirectAuthenticationHandler.class); - public static final String AUTHENTICATION_PROVIDER_URL = "authentication.provider.url"; + public static final String AUTHENTICATION_PROVIDER_URL = + "authentication.provider.url"; public static final String PUBLIC_KEY_PEM = "public.key.pem"; public static final String EXPECTED_JWT_AUDIENCES = "expected.jwt.audiences"; public static final String JWT_COOKIE_NAME = "jwt.cookie.name"; @@ -205,7 +196,6 @@ public class JWTRedirectAuthenticationHandler extends protected String getJWTFromCookie(HttpServletRequest req) { String serializedJWT = null; Cookie[] cookies = req.getCookies(); -String userName = null; if (cookies != null) { for (Cookie cookie : cookies) { if (cookieName.equals(cookie.getName())) { @@ -350,7 +340,7 @@ public class JWTRedirectAuthenticationHandler extends boolean valid = false; try { Date expires = jwtToken.getJWTClaimsSet().getExpirationTime(); - if (expires != null && new Date().before(expires)) { + if (expires == null || new Date().before(expires)) { LOG.debug("JWT token expiration date has been " + "successfully validated"); valid = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7ed05e4/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java -- diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java index 019ecb4..97a8a9d 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthentictionHandler.java
[3/3] hadoop git commit: HDFS-9034. StorageTypeStats Metric should not count failed storage. Contributed by Surendra Singh Lilhore.
HDFS-9034. StorageTypeStats Metric should not count failed storage. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a50651e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a50651e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a50651e Branch: refs/heads/branch-2.8 Commit: 5a50651e5bd7dc77fd80f79c80b2388cacd117d3 Parents: 3266709 Author: Benoy Antony Authored: Tue Dec 22 15:28:17 2015 -0800 Committer: Benoy Antony Committed: Thu Jan 14 14:42:04 2016 -0800 -- .../server/blockmanagement/DatanodeManager.java | 1 + .../server/blockmanagement/DatanodeStats.java | 19 -- .../blockmanagement/HeartbeatManager.java | 6 +- .../blockmanagement/TestBlockStatsMXBean.java | 62 4 files changed, 81 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a50651e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 59190ee..9f878d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -967,6 +967,7 @@ public class DatanodeManager { // no need to update its timestamp // because its is done when the descriptor is created heartbeatManager.addDatanode(nodeDescr); +heartbeatManager.updateDnStat(nodeDescr); incrementVersionCount(nodeReg.getSoftwareVersion()); startDecommissioningIfExcluded(nodeDescr); success = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a50651e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java index 4c39c41..bcc9bba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import java.util.EnumMap; import java.util.HashSet; @@ -61,8 +62,10 @@ class DatanodeStats { } Set storageTypes = new HashSet<>(); for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) { - statsMap.addStorage(storageInfo, node); - storageTypes.add(storageInfo.getStorageType()); + if (storageInfo.getState() != DatanodeStorage.State.FAILED) { +statsMap.addStorage(storageInfo, node); +storageTypes.add(storageInfo.getStorageType()); + } } for (StorageType storageType : storageTypes) { statsMap.addNode(storageType, node); @@ -86,8 +89,10 @@ class DatanodeStats { } Set storageTypes = new HashSet<>(); for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) { - statsMap.subtractStorage(storageInfo, node); - storageTypes.add(storageInfo.getStorageType()); + if (storageInfo.getState() != DatanodeStorage.State.FAILED) { +statsMap.subtractStorage(storageInfo, node); +storageTypes.add(storageInfo.getStorageType()); + } } for (StorageType storageType : storageTypes) { statsMap.subtractNode(storageType, node); @@ -202,10 +207,12 @@ class DatanodeStats { private void subtractNode(StorageType storageType, final DatanodeDescriptor node) { - StorageTypeStats storageTypeStats = - storageTypeStatsMap.get(storageType); + StorageTypeStats storageTypeStats = storageTypeStatsMap.get(storageType); if (storageTypeStats != null) { storageTypeStats.subtractNode(node); +if (storageTypeStats.getNodesInService() == 0) { + storageTypeStatsMap.remove(storageType); +} } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a50651e/hadoop-hdfs-project/hadoop-hdfs/src/m
[2/3] hadoop git commit: HDFS-9597. BaseReplicationPolicyTest should update data node stats after adding a data node. Contributed by Wei-Chiu Chuang.
HDFS-9597. BaseReplicationPolicyTest should update data node stats after adding a data node. Contributed by Wei-Chiu Chuang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ceee073 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ceee073 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ceee073 Branch: refs/heads/branch-2.8 Commit: 6ceee073574214502e05b10b0147a34ad910936f Parents: 5a50651 Author: Benoy Antony Authored: Fri Dec 25 08:55:39 2015 -0800 Committer: Benoy Antony Committed: Thu Jan 14 14:42:04 2016 -0800 -- .../hdfs/server/blockmanagement/BaseReplicationPolicyTest.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ceee073/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java index 6174447..7dc52fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java @@ -98,6 +98,8 @@ abstract public class BaseReplicationPolicyTest { cluster.add(dataNodes[i]); bm.getDatanodeManager().getHeartbeatManager().addDatanode( dataNodes[i]); + bm.getDatanodeManager().getHeartbeatManager().updateDnStat( + dataNodes[i]); } updateHeartbeatWithUsage(); }
[1/3] hadoop git commit: HADOOP-12587. Hadoop AuthToken refuses to work without a maxinactive attribute in issued token. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/branch-2.8 3266709b5 -> 6cfeb84a3 HADOOP-12587. Hadoop AuthToken refuses to work without a maxinactive attribute in issued token. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cfeb84a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cfeb84a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cfeb84a Branch: refs/heads/branch-2.8 Commit: 6cfeb84a3677665d608f5d3feb76efd25445a725 Parents: 6ceee07 Author: Benoy Antony Authored: Sat Jan 9 13:39:18 2016 -0800 Committer: Benoy Antony Committed: Thu Jan 14 14:42:04 2016 -0800 -- .../server/AuthenticationFilter.java| 11 +- .../security/authentication/util/AuthToken.java | 16 +- .../server/TestAuthenticationFilter.java| 167 --- .../src/site/markdown/HttpAuthentication.md | 2 +- 4 files changed, 161 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cfeb84a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index e0da38b..4bdc808 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -150,7 +150,7 @@ public class AuthenticationFilter implements Filter { * that indicates the max inactive interval of the generated token. */ public static final String - AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.MaxInactiveInterval"; + AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.max-inactive-interval"; /** * Constant for the configuration property that indicates the validity of the generated token. @@ -234,9 +234,11 @@ public class AuthenticationFilter implements Filter { } else { authHandlerClassName = authHandlerName; } - maxInactiveInterval = Long.parseLong(config.getProperty( -AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "1800")) * 1000; // 30 minutes; +AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "-1")); // By default, disable. +if (maxInactiveInterval > 0) { + maxInactiveInterval *= 1000; +} validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) * 1000; //10 hours initializeSecretProvider(filterConfig); @@ -559,7 +561,7 @@ public class AuthenticationFilter implements Filter { } token = authHandler.authenticate(httpRequest, httpResponse); if (token != null && token != AuthenticationToken.ANONYMOUS) { -if (token.getMaxInactives() != 0) { +if (token.getMaxInactives() > 0) { token.setMaxInactives(System.currentTimeMillis() + getMaxInactiveInterval() * 1000); } @@ -603,6 +605,7 @@ public class AuthenticationFilter implements Filter { && getMaxInactiveInterval() > 0) { token.setMaxInactives(System.currentTimeMillis() + getMaxInactiveInterval() * 1000); +token.setExpires(token.getExpires()); newToken = true; } if (newToken && !token.isExpired() http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cfeb84a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java index 4fbe599..e959f65 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java @@ -39,8 +39,7 @@ public class AuthToken implements Principal { private static final String TYPE = "t"; private final static Set ATTRIBUTES = -new HashSet(Arrays.asList(USER_NAME, PRINCIPAL, -MAX_INACTIVES, EXPIRES, TYPE)); + new HashSet<>(Arrays.asList(USER_NAME, PRINCIPAL, EXPIRES, TYPE)); private String u
hadoop git commit: HADOOP-12587. Hadoop AuthToken refuses to work without a maxinactive attribute in issued token. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/branch-2 80734bc34 -> 4e5f77b7f HADOOP-12587. Hadoop AuthToken refuses to work without a maxinactive attribute in issued token. (Benoy Antony) (cherry picked from commit dec8dfdfa66c37f8cc8c0900fd12f98c7529b99e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e5f77b7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e5f77b7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e5f77b7 Branch: refs/heads/branch-2 Commit: 4e5f77b7f5c9f1145743d960846233c06c304622 Parents: 80734bc Author: Benoy Antony Authored: Sat Jan 9 13:39:18 2016 -0800 Committer: Benoy Antony Committed: Sat Jan 9 13:43:15 2016 -0800 -- .../server/AuthenticationFilter.java| 11 +- .../security/authentication/util/AuthToken.java | 16 +- .../server/TestAuthenticationFilter.java| 167 --- .../src/site/markdown/HttpAuthentication.md | 2 +- 4 files changed, 161 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e5f77b7/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index e0da38b..4bdc808 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -150,7 +150,7 @@ public class AuthenticationFilter implements Filter { * that indicates the max inactive interval of the generated token. */ public static final String - AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.MaxInactiveInterval"; + AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.max-inactive-interval"; /** * Constant for the configuration property that indicates the validity of the generated token. @@ -234,9 +234,11 @@ public class AuthenticationFilter implements Filter { } else { authHandlerClassName = authHandlerName; } - maxInactiveInterval = Long.parseLong(config.getProperty( -AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "1800")) * 1000; // 30 minutes; +AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "-1")); // By default, disable. +if (maxInactiveInterval > 0) { + maxInactiveInterval *= 1000; +} validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) * 1000; //10 hours initializeSecretProvider(filterConfig); @@ -559,7 +561,7 @@ public class AuthenticationFilter implements Filter { } token = authHandler.authenticate(httpRequest, httpResponse); if (token != null && token != AuthenticationToken.ANONYMOUS) { -if (token.getMaxInactives() != 0) { +if (token.getMaxInactives() > 0) { token.setMaxInactives(System.currentTimeMillis() + getMaxInactiveInterval() * 1000); } @@ -603,6 +605,7 @@ public class AuthenticationFilter implements Filter { && getMaxInactiveInterval() > 0) { token.setMaxInactives(System.currentTimeMillis() + getMaxInactiveInterval() * 1000); +token.setExpires(token.getExpires()); newToken = true; } if (newToken && !token.isExpired() http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e5f77b7/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java index 4fbe599..e959f65 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java @@ -39,8 +39,7 @@ public class AuthToken implements Principal { private static final String TYPE = "t"; private final static Set ATTRIBUTES = -new HashSet(Arrays.asList(USER_NAME, PRINCIPAL, -MAX_INACTIVES, EXPIRES, TYPE)); + new HashSet<>(Arrays.a
hadoop git commit: HADOOP-12587. Hadoop AuthToken refuses to work without a maxinactive attribute in issued token. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/trunk d7ed04758 -> dec8dfdfa HADOOP-12587. Hadoop AuthToken refuses to work without a maxinactive attribute in issued token. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dec8dfdf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dec8dfdf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dec8dfdf Branch: refs/heads/trunk Commit: dec8dfdfa66c37f8cc8c0900fd12f98c7529b99e Parents: d7ed047 Author: Benoy Antony Authored: Sat Jan 9 13:39:18 2016 -0800 Committer: Benoy Antony Committed: Sat Jan 9 13:41:18 2016 -0800 -- .../server/AuthenticationFilter.java| 11 +- .../security/authentication/util/AuthToken.java | 16 +- .../server/TestAuthenticationFilter.java| 167 --- .../src/site/markdown/HttpAuthentication.md | 2 +- 4 files changed, 161 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dec8dfdf/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index e0da38b..4bdc808 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -150,7 +150,7 @@ public class AuthenticationFilter implements Filter { * that indicates the max inactive interval of the generated token. */ public static final String - AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.MaxInactiveInterval"; + AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.max-inactive-interval"; /** * Constant for the configuration property that indicates the validity of the generated token. @@ -234,9 +234,11 @@ public class AuthenticationFilter implements Filter { } else { authHandlerClassName = authHandlerName; } - maxInactiveInterval = Long.parseLong(config.getProperty( -AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "1800")) * 1000; // 30 minutes; +AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "-1")); // By default, disable. +if (maxInactiveInterval > 0) { + maxInactiveInterval *= 1000; +} validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) * 1000; //10 hours initializeSecretProvider(filterConfig); @@ -559,7 +561,7 @@ public class AuthenticationFilter implements Filter { } token = authHandler.authenticate(httpRequest, httpResponse); if (token != null && token != AuthenticationToken.ANONYMOUS) { -if (token.getMaxInactives() != 0) { +if (token.getMaxInactives() > 0) { token.setMaxInactives(System.currentTimeMillis() + getMaxInactiveInterval() * 1000); } @@ -603,6 +605,7 @@ public class AuthenticationFilter implements Filter { && getMaxInactiveInterval() > 0) { token.setMaxInactives(System.currentTimeMillis() + getMaxInactiveInterval() * 1000); +token.setExpires(token.getExpires()); newToken = true; } if (newToken && !token.isExpired() http://git-wip-us.apache.org/repos/asf/hadoop/blob/dec8dfdf/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java index 4fbe599..e959f65 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/AuthToken.java @@ -39,8 +39,7 @@ public class AuthToken implements Principal { private static final String TYPE = "t"; private final static Set ATTRIBUTES = -new HashSet(Arrays.asList(USER_NAME, PRINCIPAL, -MAX_INACTIVES, EXPIRES, TYPE)); + new HashSet<>(Arrays.asList(USER_NAME, PRINCIPAL, EXPIRES, TYPE)); private String u
hadoop git commit: HDFS-9597. BaseReplicationPolicyTest should update data node stats after adding a data node. Contributed by Wei-Chiu Chuang.
Repository: hadoop Updated Branches: refs/heads/branch-2 8baf9369b -> 42160d358 HDFS-9597. BaseReplicationPolicyTest should update data node stats after adding a data node. Contributed by Wei-Chiu Chuang. (cherry picked from commit 8cfd672397efd91c471c417afef3aff85f64b506) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42160d35 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42160d35 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42160d35 Branch: refs/heads/branch-2 Commit: 42160d358a8d010539603f777eafcc806d983c21 Parents: 8baf936 Author: Benoy Antony Authored: Fri Dec 25 08:55:39 2015 -0800 Committer: Benoy Antony Committed: Fri Dec 25 08:57:34 2015 -0800 -- .../hdfs/server/blockmanagement/BaseReplicationPolicyTest.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/42160d35/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java index 6174447..7dc52fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java @@ -98,6 +98,8 @@ abstract public class BaseReplicationPolicyTest { cluster.add(dataNodes[i]); bm.getDatanodeManager().getHeartbeatManager().addDatanode( dataNodes[i]); + bm.getDatanodeManager().getHeartbeatManager().updateDnStat( + dataNodes[i]); } updateHeartbeatWithUsage(); }
hadoop git commit: HDFS-9597. BaseReplicationPolicyTest should update data node stats after adding a data node. Contributed by Wei-Chiu Chuang.
Repository: hadoop Updated Branches: refs/heads/trunk 2c17b8156 -> 8cfd67239 HDFS-9597. BaseReplicationPolicyTest should update data node stats after adding a data node. Contributed by Wei-Chiu Chuang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cfd6723 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cfd6723 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cfd6723 Branch: refs/heads/trunk Commit: 8cfd672397efd91c471c417afef3aff85f64b506 Parents: 2c17b81 Author: Benoy Antony Authored: Fri Dec 25 08:55:39 2015 -0800 Committer: Benoy Antony Committed: Fri Dec 25 08:55:39 2015 -0800 -- .../hdfs/server/blockmanagement/BaseReplicationPolicyTest.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cfd6723/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java index 6174447..7dc52fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BaseReplicationPolicyTest.java @@ -98,6 +98,8 @@ abstract public class BaseReplicationPolicyTest { cluster.add(dataNodes[i]); bm.getDatanodeManager().getHeartbeatManager().addDatanode( dataNodes[i]); + bm.getDatanodeManager().getHeartbeatManager().updateDnStat( + dataNodes[i]); } updateHeartbeatWithUsage(); }
hadoop git commit: HDFS-9034. StorageTypeStats Metric should not count failed storage. Contributed by Surendra Singh Lilhore.
Repository: hadoop Updated Branches: refs/heads/branch-2 131260f0a -> 0d249a5e2 HDFS-9034. StorageTypeStats Metric should not count failed storage. Contributed by Surendra Singh Lilhore. (cherry picked from commit df83230948204ee2d2b06ecc66ce0163e2df27ef) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d249a5e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d249a5e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d249a5e Branch: refs/heads/branch-2 Commit: 0d249a5e287a54b7540f2ea7937d14d0fd44bce0 Parents: 131260f Author: Benoy Antony Authored: Tue Dec 22 15:28:17 2015 -0800 Committer: Benoy Antony Committed: Tue Dec 22 15:29:36 2015 -0800 -- .../server/blockmanagement/DatanodeManager.java | 1 + .../server/blockmanagement/DatanodeStats.java | 19 -- .../blockmanagement/HeartbeatManager.java | 6 +- .../blockmanagement/TestBlockStatsMXBean.java | 62 4 files changed, 81 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d249a5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 802bb76..d302b9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -953,6 +953,7 @@ public class DatanodeManager { // no need to update its timestamp // because its is done when the descriptor is created heartbeatManager.addDatanode(nodeDescr); +heartbeatManager.updateDnStat(nodeDescr); incrementVersionCount(nodeReg.getSoftwareVersion()); startDecommissioningIfExcluded(nodeDescr); success = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d249a5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java index 4c39c41..bcc9bba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import java.util.EnumMap; import java.util.HashSet; @@ -61,8 +62,10 @@ class DatanodeStats { } Set storageTypes = new HashSet<>(); for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) { - statsMap.addStorage(storageInfo, node); - storageTypes.add(storageInfo.getStorageType()); + if (storageInfo.getState() != DatanodeStorage.State.FAILED) { +statsMap.addStorage(storageInfo, node); +storageTypes.add(storageInfo.getStorageType()); + } } for (StorageType storageType : storageTypes) { statsMap.addNode(storageType, node); @@ -86,8 +89,10 @@ class DatanodeStats { } Set storageTypes = new HashSet<>(); for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) { - statsMap.subtractStorage(storageInfo, node); - storageTypes.add(storageInfo.getStorageType()); + if (storageInfo.getState() != DatanodeStorage.State.FAILED) { +statsMap.subtractStorage(storageInfo, node); +storageTypes.add(storageInfo.getStorageType()); + } } for (StorageType storageType : storageTypes) { statsMap.subtractNode(storageType, node); @@ -202,10 +207,12 @@ class DatanodeStats { private void subtractNode(StorageType storageType, final DatanodeDescriptor node) { - StorageTypeStats storageTypeStats = - storageTypeStatsMap.get(storageType); + StorageTypeStats storageTypeStats = storageTypeStatsMap.get(storageType); if (storageTypeStats != null) { storageTypeStats.subtractNode(node); +if (storageTypeStats.getNodesInService() == 0) { + storageType
hadoop git commit: HDFS-9034. StorageTypeStats Metric should not count failed storage. Contributed by Surendra Singh Lilhore.
Repository: hadoop Updated Branches: refs/heads/trunk 867048c3e -> df8323094 HDFS-9034. StorageTypeStats Metric should not count failed storage. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df832309 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df832309 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df832309 Branch: refs/heads/trunk Commit: df83230948204ee2d2b06ecc66ce0163e2df27ef Parents: 867048c Author: Benoy Antony Authored: Tue Dec 22 15:28:17 2015 -0800 Committer: Benoy Antony Committed: Tue Dec 22 15:28:17 2015 -0800 -- .../server/blockmanagement/DatanodeManager.java | 1 + .../server/blockmanagement/DatanodeStats.java | 19 -- .../blockmanagement/HeartbeatManager.java | 6 +- .../blockmanagement/TestBlockStatsMXBean.java | 62 4 files changed, 81 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/df832309/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 0c0c01a..53f7043 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -963,6 +963,7 @@ public class DatanodeManager { // no need to update its timestamp // because its is done when the descriptor is created heartbeatManager.addDatanode(nodeDescr); +heartbeatManager.updateDnStat(nodeDescr); incrementVersionCount(nodeReg.getSoftwareVersion()); startDecommissioningIfExcluded(nodeDescr); success = true; http://git-wip-us.apache.org/repos/asf/hadoop/blob/df832309/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java index 4c39c41..bcc9bba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import java.util.EnumMap; import java.util.HashSet; @@ -61,8 +62,10 @@ class DatanodeStats { } Set storageTypes = new HashSet<>(); for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) { - statsMap.addStorage(storageInfo, node); - storageTypes.add(storageInfo.getStorageType()); + if (storageInfo.getState() != DatanodeStorage.State.FAILED) { +statsMap.addStorage(storageInfo, node); +storageTypes.add(storageInfo.getStorageType()); + } } for (StorageType storageType : storageTypes) { statsMap.addNode(storageType, node); @@ -86,8 +89,10 @@ class DatanodeStats { } Set storageTypes = new HashSet<>(); for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) { - statsMap.subtractStorage(storageInfo, node); - storageTypes.add(storageInfo.getStorageType()); + if (storageInfo.getState() != DatanodeStorage.State.FAILED) { +statsMap.subtractStorage(storageInfo, node); +storageTypes.add(storageInfo.getStorageType()); + } } for (StorageType storageType : storageTypes) { statsMap.subtractNode(storageType, node); @@ -202,10 +207,12 @@ class DatanodeStats { private void subtractNode(StorageType storageType, final DatanodeDescriptor node) { - StorageTypeStats storageTypeStats = - storageTypeStatsMap.get(storageType); + StorageTypeStats storageTypeStats = storageTypeStatsMap.get(storageType); if (storageTypeStats != null) { storageTypeStats.subtractNode(node); +if (storageTypeStats.getNodesInService() == 0) { + storageTypeStatsMap.remove(storageType); +} } } } http://git-wip-us.a
hadoop git commit: hadoop-12050. Enable MaxInactiveInterval for hadoop http auth token. Contributed by Huizhi Lu.
Repository: hadoop Updated Branches: refs/heads/branch-2 40d8faf86 -> d84e4a90d hadoop-12050. Enable MaxInactiveInterval for hadoop http auth token. Contributed by Huizhi Lu. (cherry picked from commit 71aedfabf39e03104c8d22456e95ef6349aae6c0) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d84e4a90 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d84e4a90 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d84e4a90 Branch: refs/heads/branch-2 Commit: d84e4a90d04f85ab59913e2d7f49ba6fef5f5a94 Parents: 40d8faf Author: Benoy Antony Authored: Tue Aug 18 13:43:34 2015 -0700 Committer: Benoy Antony Committed: Tue Aug 18 13:53:47 2015 -0700 -- .../server/AuthenticationFilter.java| 63 +-- .../server/AuthenticationToken.java | 12 ++ .../security/authentication/util/AuthToken.java | 34 +++- .../server/TestAuthenticationFilter.java| 163 ++- .../src/site/markdown/HttpAuthentication.md | 8 +- 5 files changed, 258 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d84e4a90/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index bf44f48..e0da38b 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -146,6 +146,13 @@ public class AuthenticationFilter implements Filter { public static final String SIGNATURE_SECRET_FILE = SIGNATURE_SECRET + ".file"; /** + * Constant for the configuration property + * that indicates the max inactive interval of the generated token. + */ + public static final String + AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.MaxInactiveInterval"; + + /** * Constant for the configuration property that indicates the validity of the generated token. */ public static final String AUTH_TOKEN_VALIDITY = "token.validity"; @@ -190,6 +197,7 @@ public class AuthenticationFilter implements Filter { private Signer signer; private SignerSecretProvider secretProvider; private AuthenticationHandler authHandler; + private long maxInactiveInterval; private long validity; private String cookieDomain; private String cookiePath; @@ -227,6 +235,8 @@ public class AuthenticationFilter implements Filter { authHandlerClassName = authHandlerName; } +maxInactiveInterval = Long.parseLong(config.getProperty( +AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "1800")) * 1000; // 30 minutes; validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) * 1000; //10 hours initializeSecretProvider(filterConfig); @@ -355,6 +365,15 @@ public class AuthenticationFilter implements Filter { } /** + * Returns the max inactive interval time of the generated tokens. + * + * @return the max inactive interval time of the generated tokens in seconds. + */ + protected long getMaxInactiveInterval() { +return maxInactiveInterval / 1000; + } + + /** * Returns the validity time of the generated tokens. * * @return the validity time of the generated tokens, in seconds. @@ -510,8 +529,10 @@ public class AuthenticationFilter implements Filter { * @throws ServletException thrown if a processing error occurred. */ @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) - throws IOException, ServletException { + public void doFilter(ServletRequest request, + ServletResponse response, + FilterChain filterChain) + throws IOException, ServletException { boolean unauthorizedResponse = true; int errCode = HttpServletResponse.SC_UNAUTHORIZED; AuthenticationException authenticationEx = null; @@ -533,19 +554,27 @@ public class AuthenticationFilter implements Filter { if (authHandler.managementOperation(token, httpRequest, httpResponse)) { if (token == null) { if (LOG.isDebugEnabled()) { -LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest)); +LOG.debug("Request [{}] triggering authentication", +
hadoop git commit: hadoop-12050. Enable MaxInactiveInterval for hadoop http auth token. Contributed by Huizhi Lu.
Repository: hadoop Updated Branches: refs/heads/trunk 14215c8ef -> 71aedfabf hadoop-12050. Enable MaxInactiveInterval for hadoop http auth token. Contributed by Huizhi Lu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71aedfab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71aedfab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71aedfab Branch: refs/heads/trunk Commit: 71aedfabf39e03104c8d22456e95ef6349aae6c0 Parents: 14215c8 Author: Benoy Antony Authored: Tue Aug 18 13:43:34 2015 -0700 Committer: Benoy Antony Committed: Tue Aug 18 13:43:34 2015 -0700 -- .../server/AuthenticationFilter.java| 63 +-- .../server/AuthenticationToken.java | 12 ++ .../security/authentication/util/AuthToken.java | 34 +++- .../server/TestAuthenticationFilter.java| 163 ++- .../src/site/markdown/HttpAuthentication.md | 8 +- 5 files changed, 258 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/71aedfab/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index bf44f48..e0da38b 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -146,6 +146,13 @@ public class AuthenticationFilter implements Filter { public static final String SIGNATURE_SECRET_FILE = SIGNATURE_SECRET + ".file"; /** + * Constant for the configuration property + * that indicates the max inactive interval of the generated token. + */ + public static final String + AUTH_TOKEN_MAX_INACTIVE_INTERVAL = "token.MaxInactiveInterval"; + + /** * Constant for the configuration property that indicates the validity of the generated token. */ public static final String AUTH_TOKEN_VALIDITY = "token.validity"; @@ -190,6 +197,7 @@ public class AuthenticationFilter implements Filter { private Signer signer; private SignerSecretProvider secretProvider; private AuthenticationHandler authHandler; + private long maxInactiveInterval; private long validity; private String cookieDomain; private String cookiePath; @@ -227,6 +235,8 @@ public class AuthenticationFilter implements Filter { authHandlerClassName = authHandlerName; } +maxInactiveInterval = Long.parseLong(config.getProperty( +AUTH_TOKEN_MAX_INACTIVE_INTERVAL, "1800")) * 1000; // 30 minutes; validity = Long.parseLong(config.getProperty(AUTH_TOKEN_VALIDITY, "36000")) * 1000; //10 hours initializeSecretProvider(filterConfig); @@ -355,6 +365,15 @@ public class AuthenticationFilter implements Filter { } /** + * Returns the max inactive interval time of the generated tokens. + * + * @return the max inactive interval time of the generated tokens in seconds. + */ + protected long getMaxInactiveInterval() { +return maxInactiveInterval / 1000; + } + + /** * Returns the validity time of the generated tokens. * * @return the validity time of the generated tokens, in seconds. @@ -510,8 +529,10 @@ public class AuthenticationFilter implements Filter { * @throws ServletException thrown if a processing error occurred. */ @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) - throws IOException, ServletException { + public void doFilter(ServletRequest request, + ServletResponse response, + FilterChain filterChain) + throws IOException, ServletException { boolean unauthorizedResponse = true; int errCode = HttpServletResponse.SC_UNAUTHORIZED; AuthenticationException authenticationEx = null; @@ -533,19 +554,27 @@ public class AuthenticationFilter implements Filter { if (authHandler.managementOperation(token, httpRequest, httpResponse)) { if (token == null) { if (LOG.isDebugEnabled()) { -LOG.debug("Request [{}] triggering authentication", getRequestURL(httpRequest)); +LOG.debug("Request [{}] triggering authentication", +getRequestURL(httpRequest)); } token =
hadoop git commit: HADOOP-12049. Control http authentication cookie persistence via configuration. Contributed by Huizhi Lu.
Repository: hadoop Updated Branches: refs/heads/branch-2 431f68530 -> aef9ab212 HADOOP-12049. Control http authentication cookie persistence via configuration. Contributed by Huizhi Lu. (cherry picked from commit a815cc157ceb24e02189634a85abed8e874568e0) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aef9ab21 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aef9ab21 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aef9ab21 Branch: refs/heads/branch-2 Commit: aef9ab2128ec6e2bb30dd41c3850c80eea5ed6bd Parents: 431f685 Author: Benoy Antony Authored: Wed Jun 24 15:59:39 2015 -0700 Committer: Benoy Antony Committed: Wed Jun 24 16:41:23 2015 -0700 -- .../server/AuthenticationFilter.java| 28 ++- .../http/TestAuthenticationSessionCookie.java | 187 +++ .../apache/hadoop/http/TestHttpCookieFlag.java | 2 +- 3 files changed, 213 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aef9ab21/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 0f86623..bf44f48 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -161,6 +161,12 @@ public class AuthenticationFilter implements Filter { public static final String COOKIE_PATH = "cookie.path"; /** + * Constant for the configuration property + * that indicates the persistence of the HTTP cookie. + */ + public static final String COOKIE_PERSISTENT = "cookie.persistent"; + + /** * Constant for the configuration property that indicates the name of the * SignerSecretProvider class to use. * Possible values are: "string", "random", "zookeeper", or a classname. @@ -187,6 +193,7 @@ public class AuthenticationFilter implements Filter { private long validity; private String cookieDomain; private String cookiePath; + private boolean isCookiePersistent; private boolean isInitializedByTomcat; /** @@ -228,6 +235,9 @@ public class AuthenticationFilter implements Filter { cookieDomain = config.getProperty(COOKIE_DOMAIN, null); cookiePath = config.getProperty(COOKIE_PATH, null); +isCookiePersistent = Boolean.parseBoolean( +config.getProperty(COOKIE_PERSISTENT, "false")); + } protected void initializeAuthHandler(String authHandlerClassName, FilterConfig filterConfig) @@ -372,6 +382,15 @@ public class AuthenticationFilter implements Filter { } /** + * Returns the cookie persistence to use for the HTTP cookie. + * + * @return the cookie persistence to use for the HTTP cookie. + */ + protected boolean isCookiePersistent() { +return isCookiePersistent; + } + + /** * Destroys the filter. * * It invokes the {@link AuthenticationHandler#destroy()} method to release any resources it may hold. @@ -549,7 +568,8 @@ public class AuthenticationFilter implements Filter { if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) { String signedToken = signer.sign(token.toString()); createAuthCookie(httpResponse, signedToken, getCookieDomain(), -getCookiePath(), token.getExpires(), isHttps); +getCookiePath(), token.getExpires(), +isCookiePersistent(), isHttps); } doFilter(filterChain, httpRequest, httpResponse); } @@ -569,7 +589,7 @@ public class AuthenticationFilter implements Filter { if (unauthorizedResponse) { if (!httpResponse.isCommitted()) { createAuthCookie(httpResponse, "", getCookieDomain(), -getCookiePath(), 0, isHttps); +getCookiePath(), 0, isCookiePersistent(), isHttps); // If response code is 401. Then WWW-Authenticate Header should be // present.. reset to 403 if not found.. if ((errCode == HttpServletResponse.SC_UNAUTHORIZED) @@ -614,6 +634,7 @@ public class AuthenticationFilter implements Filter { * @param isSecure is the cookie secure? * @param token the token. * @param expires the cookie expiration time. + * @param isCoo
hadoop git commit: HADOOP-12049. Control http authentication cookie persistence via configuration. Contributed by Huizhi Lu.
Repository: hadoop Updated Branches: refs/heads/trunk afe9ea3c1 -> a815cc157 HADOOP-12049. Control http authentication cookie persistence via configuration. Contributed by Huizhi Lu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a815cc15 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a815cc15 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a815cc15 Branch: refs/heads/trunk Commit: a815cc157ceb24e02189634a85abed8e874568e0 Parents: afe9ea3 Author: Benoy Antony Authored: Wed Jun 24 15:59:39 2015 -0700 Committer: Benoy Antony Committed: Wed Jun 24 15:59:39 2015 -0700 -- .../server/AuthenticationFilter.java| 28 ++- .../http/TestAuthenticationSessionCookie.java | 187 +++ .../apache/hadoop/http/TestHttpCookieFlag.java | 2 +- 3 files changed, 213 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a815cc15/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 0f86623..bf44f48 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -161,6 +161,12 @@ public class AuthenticationFilter implements Filter { public static final String COOKIE_PATH = "cookie.path"; /** + * Constant for the configuration property + * that indicates the persistence of the HTTP cookie. + */ + public static final String COOKIE_PERSISTENT = "cookie.persistent"; + + /** * Constant for the configuration property that indicates the name of the * SignerSecretProvider class to use. * Possible values are: "string", "random", "zookeeper", or a classname. @@ -187,6 +193,7 @@ public class AuthenticationFilter implements Filter { private long validity; private String cookieDomain; private String cookiePath; + private boolean isCookiePersistent; private boolean isInitializedByTomcat; /** @@ -228,6 +235,9 @@ public class AuthenticationFilter implements Filter { cookieDomain = config.getProperty(COOKIE_DOMAIN, null); cookiePath = config.getProperty(COOKIE_PATH, null); +isCookiePersistent = Boolean.parseBoolean( +config.getProperty(COOKIE_PERSISTENT, "false")); + } protected void initializeAuthHandler(String authHandlerClassName, FilterConfig filterConfig) @@ -372,6 +382,15 @@ public class AuthenticationFilter implements Filter { } /** + * Returns the cookie persistence to use for the HTTP cookie. + * + * @return the cookie persistence to use for the HTTP cookie. + */ + protected boolean isCookiePersistent() { +return isCookiePersistent; + } + + /** * Destroys the filter. * * It invokes the {@link AuthenticationHandler#destroy()} method to release any resources it may hold. @@ -549,7 +568,8 @@ public class AuthenticationFilter implements Filter { if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) { String signedToken = signer.sign(token.toString()); createAuthCookie(httpResponse, signedToken, getCookieDomain(), -getCookiePath(), token.getExpires(), isHttps); +getCookiePath(), token.getExpires(), +isCookiePersistent(), isHttps); } doFilter(filterChain, httpRequest, httpResponse); } @@ -569,7 +589,7 @@ public class AuthenticationFilter implements Filter { if (unauthorizedResponse) { if (!httpResponse.isCommitted()) { createAuthCookie(httpResponse, "", getCookieDomain(), -getCookiePath(), 0, isHttps); +getCookiePath(), 0, isCookiePersistent(), isHttps); // If response code is 401. Then WWW-Authenticate Header should be // present.. reset to 403 if not found.. if ((errCode == HttpServletResponse.SC_UNAUTHORIZED) @@ -614,6 +634,7 @@ public class AuthenticationFilter implements Filter { * @param isSecure is the cookie secure? * @param token the token. * @param expires the cookie expiration time. + * @param isCookiePersistent whether the cookie is persistent or not. * * XXX the followin
hadoop git commit: HDFS-7467. Provide storage tier information for a directory via fsck. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/branch-2 d389a1ae9 -> 4e400030f HDFS-7467. Provide storage tier information for a directory via fsck. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e400030 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e400030 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e400030 Branch: refs/heads/branch-2 Commit: 4e400030f6bfb3cf2c689aefb526d5296ee077ec Parents: d389a1a Author: Benoy Antony Authored: Wed Feb 25 16:19:35 2015 -0800 Committer: Benoy Antony Committed: Wed Feb 25 16:46:41 2015 -0800 -- .../hdfs/server/namenode/NamenodeFsck.java | 23 +- .../server/namenode/StoragePolicySummary.java | 257 +++ .../org/apache/hadoop/hdfs/tools/DFSck.java | 4 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 78 +- .../namenode/TestStoragePolicySummary.java | 201 +++ 5 files changed, 549 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e400030/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index f96618b..8133529 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Iterator; @@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.RemotePeerFactory; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; @@ -127,6 +129,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { private boolean showBlocks = false; private boolean showLocations = false; private boolean showRacks = false; + private boolean showStoragePolcies = false; private boolean showCorruptFileBlocks = false; /** @@ -163,6 +166,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { private List snapshottableDirs = null; private final BlockPlacementPolicy bpPolicy; + private StoragePolicySummary storageTypeSummary = null; /** * Filesystem checker. @@ -199,6 +203,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { else if (key.equals("blocks")) { this.showBlocks = true; } else if (key.equals("locations")) { this.showLocations = true; } else if (key.equals("racks")) { this.showRacks = true; } + else if (key.equals("storagepolicies")) { this.showStoragePolcies = true; } else if (key.equals("openforwrite")) {this.showOpenFiles = true; } else if (key.equals("listcorruptfileblocks")) { this.showCorruptFileBlocks = true; @@ -332,6 +337,11 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { return; } +if (this.showStoragePolcies) { + storageTypeSummary = new StoragePolicySummary( + namenode.getNamesystem().getBlockManager().getStoragePolicies()); +} + Result res = new Result(conf); check(path, file, res); @@ -340,6 +350,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { out.println(" Number of data-nodes:\t\t" + totalDatanodes); out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks()); +if (this.showStoragePolcies) { + out.print(storageTypeSummary.toString()); +} + out.println("FSCK ended at " + new Date() + " in " + (Time.now() - startTime + " milliseconds")); @@ -487,7 +501,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { boolean isCorrupt = lBlk.isCorrupt(); String blkName = block.toString(); DatanodeInfo[] locs = lBlk.getLocations(); - NumberReplicas numberReplicas = namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock()); + NumberReplicas numberReplicas =
hadoop git commit: HDFS-7467. Provide storage tier information for a directory via fsck. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/trunk caa42adf2 -> d140d76a4 HDFS-7467. Provide storage tier information for a directory via fsck. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d140d76a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d140d76a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d140d76a Branch: refs/heads/trunk Commit: d140d76a43c88e326b9c2818578f22bd3563b969 Parents: caa42ad Author: Benoy Antony Authored: Wed Feb 25 16:19:35 2015 -0800 Committer: Benoy Antony Committed: Wed Feb 25 16:19:35 2015 -0800 -- .../hdfs/server/namenode/NamenodeFsck.java | 23 +- .../server/namenode/StoragePolicySummary.java | 257 +++ .../org/apache/hadoop/hdfs/tools/DFSck.java | 2 + .../hadoop/hdfs/server/namenode/TestFsck.java | 78 +- .../namenode/TestStoragePolicySummary.java | 201 +++ 5 files changed, 548 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d140d76a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index dc9494d..5134f3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.Iterator; @@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.RemotePeerFactory; +import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; @@ -127,6 +129,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { private boolean showBlocks = false; private boolean showLocations = false; private boolean showRacks = false; + private boolean showStoragePolcies = false; private boolean showprogress = false; private boolean showCorruptFileBlocks = false; @@ -164,6 +167,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { private List snapshottableDirs = null; private final BlockPlacementPolicy bpPolicy; + private StoragePolicySummary storageTypeSummary = null; /** * Filesystem checker. @@ -200,6 +204,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { else if (key.equals("blocks")) { this.showBlocks = true; } else if (key.equals("locations")) { this.showLocations = true; } else if (key.equals("racks")) { this.showRacks = true; } + else if (key.equals("storagepolicies")) { this.showStoragePolcies = true; } else if (key.equals("showprogress")) { this.showprogress = true; } else if (key.equals("openforwrite")) {this.showOpenFiles = true; } else if (key.equals("listcorruptfileblocks")) { @@ -334,6 +339,11 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { return; } +if (this.showStoragePolcies) { + storageTypeSummary = new StoragePolicySummary( + namenode.getNamesystem().getBlockManager().getStoragePolicies()); +} + Result res = new Result(conf); check(path, file, res); @@ -342,6 +352,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { out.println(" Number of data-nodes:\t\t" + totalDatanodes); out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks()); +if (this.showStoragePolcies) { + out.print(storageTypeSummary.toString()); +} + out.println("FSCK ended at " + new Date() + " in " + (Time.now() - startTime + " milliseconds")); @@ -492,7 +506,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { boolean isCorrupt = lBlk.isCorrupt(); String blkName = block.toString(); DatanodeInfo[] locs = lBlk.getLocations(); - NumberReplicas numberReplicas = namenode.getNamesystem().getBlockManager().countNodes(block
hadoop git commit: HADOOP-11494. Lock acquisition on WrappedInputStream#unwrappedRpcBuffer may race with another thread. Contributed by Ted Yu.
Repository: hadoop Updated Branches: refs/heads/branch-2 648510e03 -> f3ed5b68c HADOOP-11494. Lock acquisition on WrappedInputStream#unwrappedRpcBuffer may race with another thread. Contributed by Ted Yu. (cherry picked from commit 3472e3bd6c50558870b86c9ccfea5072385fa991) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3ed5b68 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3ed5b68 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3ed5b68 Branch: refs/heads/branch-2 Commit: f3ed5b68ca0963290298c180db9ab9dd20a6dd2d Parents: 648510e Author: Benoy Antony Authored: Mon Feb 2 10:34:47 2015 -0800 Committer: Benoy Antony Committed: Mon Feb 2 10:46:54 2015 -0800 -- .../org/apache/hadoop/security/SaslRpcClient.java | 18 -- 1 file changed, 8 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ed5b68/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java index dfb0898..4a1a397 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java @@ -573,17 +573,15 @@ public class SaslRpcClient { } @Override -public int read(byte[] buf, int off, int len) throws IOException { - synchronized(unwrappedRpcBuffer) { -// fill the buffer with the next RPC message -if (unwrappedRpcBuffer.remaining() == 0) { - readNextRpcPacket(); -} -// satisfy as much of the request as possible -int readLen = Math.min(len, unwrappedRpcBuffer.remaining()); -unwrappedRpcBuffer.get(buf, off, readLen); -return readLen; +public synchronized int read(byte[] buf, int off, int len) throws IOException { + // fill the buffer with the next RPC message + if (unwrappedRpcBuffer.remaining() == 0) { +readNextRpcPacket(); } + // satisfy as much of the request as possible + int readLen = Math.min(len, unwrappedRpcBuffer.remaining()); + unwrappedRpcBuffer.get(buf, off, readLen); + return readLen; } // all messages must be RPC SASL wrapped, else an exception is thrown
hadoop git commit: HADOOP-11494. Lock acquisition on WrappedInputStream#unwrappedRpcBuffer may race with another thread. Contributed by Ted Yu.
Repository: hadoop Updated Branches: refs/heads/trunk ffc75d6eb -> 3472e3bd6 HADOOP-11494. Lock acquisition on WrappedInputStream#unwrappedRpcBuffer may race with another thread. Contributed by Ted Yu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3472e3bd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3472e3bd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3472e3bd Branch: refs/heads/trunk Commit: 3472e3bd6c50558870b86c9ccfea5072385fa991 Parents: ffc75d6 Author: Benoy Antony Authored: Mon Feb 2 10:34:47 2015 -0800 Committer: Benoy Antony Committed: Mon Feb 2 10:34:47 2015 -0800 -- .../org/apache/hadoop/security/SaslRpcClient.java | 18 -- 1 file changed, 8 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3472e3bd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java index dfb0898..4a1a397 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java @@ -573,17 +573,15 @@ public class SaslRpcClient { } @Override -public int read(byte[] buf, int off, int len) throws IOException { - synchronized(unwrappedRpcBuffer) { -// fill the buffer with the next RPC message -if (unwrappedRpcBuffer.remaining() == 0) { - readNextRpcPacket(); -} -// satisfy as much of the request as possible -int readLen = Math.min(len, unwrappedRpcBuffer.remaining()); -unwrappedRpcBuffer.get(buf, off, readLen); -return readLen; +public synchronized int read(byte[] buf, int off, int len) throws IOException { + // fill the buffer with the next RPC message + if (unwrappedRpcBuffer.remaining() == 0) { +readNextRpcPacket(); } + // satisfy as much of the request as possible + int readLen = Math.min(len, unwrappedRpcBuffer.remaining()); + unwrappedRpcBuffer.get(buf, off, readLen); + return readLen; } // all messages must be RPC SASL wrapped, else an exception is thrown
hadoop git commit: HADOOP-10651. Add ability to restrict service access using IP addresses and hostnames. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/branch-2 d2fbba790 -> 832ae27f8 HADOOP-10651. Add ability to restrict service access using IP addresses and hostnames. (Benoy Antony) (cherry picked from commit 20625c8f048701c9516da159b24c0b33983e4bb7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/832ae27f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/832ae27f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/832ae27f Branch: refs/heads/branch-2 Commit: 832ae27f83856dfec1eb8f898fb1f9cfd04a1877 Parents: d2fbba7 Author: Benoy Antony Authored: Thu Jan 8 10:06:48 2015 -0800 Committer: Benoy Antony Committed: Thu Jan 8 10:09:55 2015 -0800 -- .../authorize/ServiceAuthorizationManager.java | 58 +- .../org/apache/hadoop/util/MachineList.java | 3 +- .../src/site/apt/ServiceLevelAuth.apt.vm| 25 +++ .../authorize/TestServiceAuthorization.java | 189 +-- 4 files changed, 259 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/832ae27f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index 272538a..5d29516 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.MachineList; import com.google.common.annotations.VisibleForTesting; @@ -44,6 +45,7 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceStability.Evolving public class ServiceAuthorizationManager { static final String BLOCKED = ".blocked"; + static final String HOSTS = ".hosts"; private static final String HADOOP_POLICY_FILE = "hadoop-policy.xml"; @@ -51,6 +53,10 @@ public class ServiceAuthorizationManager { // and second ACL specifies blocked entries. private volatile Map, AccessControlList[]> protocolToAcls = new IdentityHashMap, AccessControlList[]>(); + // For each class, first MachineList in the array specifies the allowed entries + // and second MachineList specifies blocked entries. + private volatile Map, MachineList[]> protocolToMachineLists = +new IdentityHashMap, MachineList[]>(); /** * Configuration key for controlling service-level authorization for Hadoop. @@ -85,7 +91,8 @@ public class ServiceAuthorizationManager { InetAddress addr ) throws AuthorizationException { AccessControlList[] acls = protocolToAcls.get(protocol); -if (acls == null) { +MachineList[] hosts = protocolToMachineLists.get(protocol); +if (acls == null || hosts == null) { throw new AuthorizationException("Protocol " + protocol + " is not known."); } @@ -115,6 +122,16 @@ public class ServiceAuthorizationManager { " is not authorized for protocol " + protocol + ", expected client Kerberos principal is " + clientPrincipal); } +if (addr != null) { + String hostAddress = addr.getHostAddress(); + if (hosts.length != 2 || !hosts[0].includes(hostAddress) || + hosts[1].includes(hostAddress)) { +AUDITLOG.warn(AUTHZ_FAILED_FOR + " for protocol=" + protocol ++ " from host = " + hostAddress); +throw new AuthorizationException("Host " + hostAddress + +" is not authorized for protocol " + protocol) ; + } +} AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol); } @@ -135,6 +152,8 @@ public class ServiceAuthorizationManager { PolicyProvider provider) { final Map, AccessControlList[]> newAcls = new IdentityHashMap, AccessControlList[]>(); +final Map, MachineList[]> newMachineLists = + new IdentityHashMap, MachineList[]>(); String defaultAcl = conf.get( CommonConfigurationKeys.HADOOP_S
hadoop git commit: HADOOP-10651. Add ability to restrict service access using IP addresses and hostnames. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/trunk dc2eaa26b -> 20625c8f0 HADOOP-10651. Add ability to restrict service access using IP addresses and hostnames. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20625c8f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20625c8f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20625c8f Branch: refs/heads/trunk Commit: 20625c8f048701c9516da159b24c0b33983e4bb7 Parents: dc2eaa2 Author: Benoy Antony Authored: Thu Jan 8 10:06:48 2015 -0800 Committer: Benoy Antony Committed: Thu Jan 8 10:06:48 2015 -0800 -- .../authorize/ServiceAuthorizationManager.java | 58 +- .../org/apache/hadoop/util/MachineList.java | 3 +- .../src/site/apt/ServiceLevelAuth.apt.vm| 25 +++ .../authorize/TestServiceAuthorization.java | 189 +-- 4 files changed, 259 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/20625c8f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index 272538a..5d29516 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.MachineList; import com.google.common.annotations.VisibleForTesting; @@ -44,6 +45,7 @@ import com.google.common.annotations.VisibleForTesting; @InterfaceStability.Evolving public class ServiceAuthorizationManager { static final String BLOCKED = ".blocked"; + static final String HOSTS = ".hosts"; private static final String HADOOP_POLICY_FILE = "hadoop-policy.xml"; @@ -51,6 +53,10 @@ public class ServiceAuthorizationManager { // and second ACL specifies blocked entries. private volatile Map, AccessControlList[]> protocolToAcls = new IdentityHashMap, AccessControlList[]>(); + // For each class, first MachineList in the array specifies the allowed entries + // and second MachineList specifies blocked entries. + private volatile Map, MachineList[]> protocolToMachineLists = +new IdentityHashMap, MachineList[]>(); /** * Configuration key for controlling service-level authorization for Hadoop. @@ -85,7 +91,8 @@ public class ServiceAuthorizationManager { InetAddress addr ) throws AuthorizationException { AccessControlList[] acls = protocolToAcls.get(protocol); -if (acls == null) { +MachineList[] hosts = protocolToMachineLists.get(protocol); +if (acls == null || hosts == null) { throw new AuthorizationException("Protocol " + protocol + " is not known."); } @@ -115,6 +122,16 @@ public class ServiceAuthorizationManager { " is not authorized for protocol " + protocol + ", expected client Kerberos principal is " + clientPrincipal); } +if (addr != null) { + String hostAddress = addr.getHostAddress(); + if (hosts.length != 2 || !hosts[0].includes(hostAddress) || + hosts[1].includes(hostAddress)) { +AUDITLOG.warn(AUTHZ_FAILED_FOR + " for protocol=" + protocol ++ " from host = " + hostAddress); +throw new AuthorizationException("Host " + hostAddress + +" is not authorized for protocol " + protocol) ; + } +} AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol); } @@ -135,6 +152,8 @@ public class ServiceAuthorizationManager { PolicyProvider provider) { final Map, AccessControlList[]> newAcls = new IdentityHashMap, AccessControlList[]>(); +final Map, MachineList[]> newMachineLists = + new IdentityHashMap, MachineList[]>(); String defaultAcl = conf.get( CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL, @@ -143,6 +162,13 @@ public class ServiceAuthorizationManager {
hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.
Repository: hadoop Updated Branches: refs/heads/branch-2 d070597a0 -> 0c2d996c2 HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena. (cherry picked from commit 53caeaa16b1450b54e994c77f5d0c8a767b88d57) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c2d996c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c2d996c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c2d996c Branch: refs/heads/branch-2 Commit: 0c2d996c2ce2f3e1c959a04f15bb6fb0ba9c4b1f Parents: d070597 Author: Benoy Antony Authored: Mon Jan 5 15:06:46 2015 -0800 Committer: Benoy Antony Committed: Mon Jan 5 15:09:19 2015 -0800 -- .../java/org/apache/hadoop/security/Groups.java | 36 +-- .../hadoop/security/TestGroupsCaching.java | 48 2 files changed, 71 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2d996c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index f3c5094..9fd39b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -23,12 +23,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ticker; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.Cache; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -60,14 +62,13 @@ public class Groups { private final GroupMappingServiceProvider impl; private final LoadingCache> cache; - private final ConcurrentHashMap negativeCacheMask = -new ConcurrentHashMap(); private final Map> staticUserToGroupsMap = new HashMap>(); private final long cacheTimeout; private final long negativeCacheTimeout; private final long warningDeltaMs; private final Timer timer; + private Set negativeCache; public Groups(Configuration conf) { this(conf, new Timer()); @@ -99,11 +100,24 @@ public class Groups { .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS) .build(new GroupCacheLoader()); +if(negativeCacheTimeout > 0) { + Cache tempMap = CacheBuilder.newBuilder() +.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS) +.ticker(new TimerToTickerAdapter(timer)) +.build(); + negativeCache = Collections.newSetFromMap(tempMap.asMap()); +} + if(LOG.isDebugEnabled()) LOG.debug("Group mapping impl=" + impl.getClass().getName() + "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" + warningDeltaMs); } + + @VisibleForTesting + Set getNegativeCache() { +return negativeCache; + } /* * Parse the hadoop.user.group.static.mapping.overrides configuration to @@ -159,13 +173,8 @@ public class Groups { // Check the negative cache first if (isNegativeCacheEnabled()) { - Long expirationTime = negativeCacheMask.get(user); - if (expirationTime != null) { -if (timer.monotonicNow() < expirationTime) { - throw noGroupsForUser(user); -} else { - negativeCacheMask.remove(user, expirationTime); -} + if (negativeCache.contains(user)) { +throw noGroupsForUser(user); } } @@ -212,8 +221,7 @@ public class Groups { if (groups.isEmpty()) { if (isNegativeCacheEnabled()) { - long expirationTime = timer.monotonicNow() + negativeCacheTimeout; - negativeCacheMask.put(user, expirationTime); + negativeCache.add(user); } // We throw here to prevent Cache from retaining an empty group @@ -252,7 +260,9 @@ public class Groups { LOG.warn("Error refreshing groups cache", e); } cache.invalidateAll(); -negativeCacheMask.clear(); +if(isNegativeCacheEnabled()) { + negativeCache.clear(); +} } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2d996c/hadoop-common-proj
hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.
Repository: hadoop Updated Branches: refs/heads/trunk 94d342e60 -> 53caeaa16 HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53caeaa1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53caeaa1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53caeaa1 Branch: refs/heads/trunk Commit: 53caeaa16b1450b54e994c77f5d0c8a767b88d57 Parents: 94d342e Author: Benoy Antony Authored: Mon Jan 5 15:06:46 2015 -0800 Committer: Benoy Antony Committed: Mon Jan 5 15:06:46 2015 -0800 -- .../java/org/apache/hadoop/security/Groups.java | 36 +-- .../hadoop/security/TestGroupsCaching.java | 48 2 files changed, 71 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53caeaa1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java index f3c5094..9fd39b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java @@ -23,12 +23,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ticker; import com.google.common.cache.CacheBuilder; +import com.google.common.cache.Cache; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -60,14 +62,13 @@ public class Groups { private final GroupMappingServiceProvider impl; private final LoadingCache> cache; - private final ConcurrentHashMap negativeCacheMask = -new ConcurrentHashMap(); private final Map> staticUserToGroupsMap = new HashMap>(); private final long cacheTimeout; private final long negativeCacheTimeout; private final long warningDeltaMs; private final Timer timer; + private Set negativeCache; public Groups(Configuration conf) { this(conf, new Timer()); @@ -99,11 +100,24 @@ public class Groups { .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS) .build(new GroupCacheLoader()); +if(negativeCacheTimeout > 0) { + Cache tempMap = CacheBuilder.newBuilder() +.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS) +.ticker(new TimerToTickerAdapter(timer)) +.build(); + negativeCache = Collections.newSetFromMap(tempMap.asMap()); +} + if(LOG.isDebugEnabled()) LOG.debug("Group mapping impl=" + impl.getClass().getName() + "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" + warningDeltaMs); } + + @VisibleForTesting + Set getNegativeCache() { +return negativeCache; + } /* * Parse the hadoop.user.group.static.mapping.overrides configuration to @@ -159,13 +173,8 @@ public class Groups { // Check the negative cache first if (isNegativeCacheEnabled()) { - Long expirationTime = negativeCacheMask.get(user); - if (expirationTime != null) { -if (timer.monotonicNow() < expirationTime) { - throw noGroupsForUser(user); -} else { - negativeCacheMask.remove(user, expirationTime); -} + if (negativeCache.contains(user)) { +throw noGroupsForUser(user); } } @@ -212,8 +221,7 @@ public class Groups { if (groups.isEmpty()) { if (isNegativeCacheEnabled()) { - long expirationTime = timer.monotonicNow() + negativeCacheTimeout; - negativeCacheMask.put(user, expirationTime); + negativeCache.add(user); } // We throw here to prevent Cache from retaining an empty group @@ -252,7 +260,9 @@ public class Groups { LOG.warn("Error refreshing groups cache", e); } cache.invalidateAll(); -negativeCacheMask.clear(); +if(isNegativeCacheEnabled()) { + negativeCache.clear(); +} } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/53caeaa1/hadoop-common-project/hadoop-common/src/t
hadoop git commit: HADOOP-10852 Fix thread safety issues in NetgroupCache. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/branch-2 b1ccf68db -> 2eabbae29 HADOOP-10852 Fix thread safety issues in NetgroupCache. (Benoy Antony) (cherry picked from commit a095622f36c5e9fff3ec02b14b800038a81f6286) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2eabbae2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2eabbae2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2eabbae2 Branch: refs/heads/branch-2 Commit: 2eabbae29288dd597395648529f4db04d95265cf Parents: b1ccf68 Author: Benoy Antony Authored: Mon Dec 15 14:00:25 2014 -0800 Committer: Benoy Antony Committed: Mon Dec 15 14:02:15 2014 -0800 -- .../apache/hadoop/security/NetgroupCache.java | 61 +++- 1 file changed, 33 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2eabbae2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java index bd9c448..4495a66 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.security; +import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.HashSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience; @@ -36,14 +36,9 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Unstable public class NetgroupCache { - private static boolean netgroupToUsersMapUpdated = true; - private static Map> netgroupToUsersMap = -new ConcurrentHashMap>(); - - private static Map> userToNetgroupsMap = + private static ConcurrentHashMap> userToNetgroupsMap = new ConcurrentHashMap>(); - /** * Get netgroups for a given user * @@ -52,21 +47,11 @@ public class NetgroupCache { */ public static void getNetgroups(final String user, List groups) { -if(netgroupToUsersMapUpdated) { - netgroupToUsersMapUpdated = false; // at the beginning to avoid race - //update userToNetgroupsMap - for(String netgroup : netgroupToUsersMap.keySet()) { -for(String netuser : netgroupToUsersMap.get(netgroup)) { - // add to userToNetgroupsMap - if(!userToNetgroupsMap.containsKey(netuser)) { -userToNetgroupsMap.put(netuser, new HashSet()); - } - userToNetgroupsMap.get(netuser).add(netgroup); -} - } -} -if(userToNetgroupsMap.containsKey(user)) { - groups.addAll(userToNetgroupsMap.get(user)); +Set userGroups = userToNetgroupsMap.get(user); +//ConcurrentHashMap does not allow null values; +//So null value check can be used to check if the key exists +if (userGroups != null) { + groups.addAll(userGroups); } } @@ -76,7 +61,15 @@ public class NetgroupCache { * @return list of cached groups */ public static List getNetgroupNames() { -return new LinkedList(netgroupToUsersMap.keySet()); +return new LinkedList(getGroups()); + } + + private static Set getGroups() { +Set allGroups = new HashSet (); +for (Set userGroups : userToNetgroupsMap.values()) { + allGroups.addAll(userGroups); +} +return allGroups; } /** @@ -86,14 +79,13 @@ public class NetgroupCache { * @return true if group is cached, false otherwise */ public static boolean isCached(String group) { -return netgroupToUsersMap.containsKey(group); +return getGroups().contains(group); } /** * Clear the cache */ public static void clear() { -netgroupToUsersMap.clear(); userToNetgroupsMap.clear(); } @@ -104,7 +96,20 @@ public class NetgroupCache { * @param users list of users for a given group */ public static void add(String group, List users) { -netgroupToUsersMap.put(group, new HashSet(users)); -netgroupToUsersMapUpdated = true; // at the end to avoid race +for (String user : users) { + Set userGroups = userToNetgroupsMap.get(user); + // ConcurrentHashMap does not allow null values; + // So null value check can be used to check if the key exists + if (userGroups == null) { +//Generate a Conc
hadoop git commit: HADOOP-10852 Fix thread safety issues in NetgroupCache. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/trunk e597249d3 -> a095622f3 HADOOP-10852 Fix thread safety issues in NetgroupCache. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a095622f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a095622f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a095622f Branch: refs/heads/trunk Commit: a095622f36c5e9fff3ec02b14b800038a81f6286 Parents: e597249 Author: Benoy Antony Authored: Mon Dec 15 14:00:25 2014 -0800 Committer: Benoy Antony Committed: Mon Dec 15 14:00:25 2014 -0800 -- .../apache/hadoop/security/NetgroupCache.java | 61 +++- 1 file changed, 33 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a095622f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java index bd9c448..4495a66 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/NetgroupCache.java @@ -17,11 +17,11 @@ */ package org.apache.hadoop.security; +import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.HashSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience; @@ -36,14 +36,9 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Unstable public class NetgroupCache { - private static boolean netgroupToUsersMapUpdated = true; - private static Map> netgroupToUsersMap = -new ConcurrentHashMap>(); - - private static Map> userToNetgroupsMap = + private static ConcurrentHashMap> userToNetgroupsMap = new ConcurrentHashMap>(); - /** * Get netgroups for a given user * @@ -52,21 +47,11 @@ public class NetgroupCache { */ public static void getNetgroups(final String user, List groups) { -if(netgroupToUsersMapUpdated) { - netgroupToUsersMapUpdated = false; // at the beginning to avoid race - //update userToNetgroupsMap - for(String netgroup : netgroupToUsersMap.keySet()) { -for(String netuser : netgroupToUsersMap.get(netgroup)) { - // add to userToNetgroupsMap - if(!userToNetgroupsMap.containsKey(netuser)) { -userToNetgroupsMap.put(netuser, new HashSet()); - } - userToNetgroupsMap.get(netuser).add(netgroup); -} - } -} -if(userToNetgroupsMap.containsKey(user)) { - groups.addAll(userToNetgroupsMap.get(user)); +Set userGroups = userToNetgroupsMap.get(user); +//ConcurrentHashMap does not allow null values; +//So null value check can be used to check if the key exists +if (userGroups != null) { + groups.addAll(userGroups); } } @@ -76,7 +61,15 @@ public class NetgroupCache { * @return list of cached groups */ public static List getNetgroupNames() { -return new LinkedList(netgroupToUsersMap.keySet()); +return new LinkedList(getGroups()); + } + + private static Set getGroups() { +Set allGroups = new HashSet (); +for (Set userGroups : userToNetgroupsMap.values()) { + allGroups.addAll(userGroups); +} +return allGroups; } /** @@ -86,14 +79,13 @@ public class NetgroupCache { * @return true if group is cached, false otherwise */ public static boolean isCached(String group) { -return netgroupToUsersMap.containsKey(group); +return getGroups().contains(group); } /** * Clear the cache */ public static void clear() { -netgroupToUsersMap.clear(); userToNetgroupsMap.clear(); } @@ -104,7 +96,20 @@ public class NetgroupCache { * @param users list of users for a given group */ public static void add(String group, List users) { -netgroupToUsersMap.put(group, new HashSet(users)); -netgroupToUsersMapUpdated = true; // at the end to avoid race +for (String user : users) { + Set userGroups = userToNetgroupsMap.get(user); + // ConcurrentHashMap does not allow null values; + // So null value check can be used to check if the key exists + if (userGroups == null) { +//Generate a ConcurrentHashSet (backed by the keyset of the
git commit: HDFS-7184. Allow data migration tool to run as a daemon. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/trunk d5084b9fa -> e4d6a8785 HDFS-7184. Allow data migration tool to run as a daemon. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4d6a878 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4d6a878 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4d6a878 Branch: refs/heads/trunk Commit: e4d6a878541cc07fada2bd07dedc4740570a472e Parents: d5084b9 Author: Benoy Antony Authored: Mon Oct 20 12:15:58 2014 -0700 Committer: Benoy Antony Committed: Mon Oct 20 12:15:58 2014 -0700 -- hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d6a878/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index a5cb73c..e88ae08 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -164,6 +164,7 @@ case ${COMMAND} in CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; mover) +daemon="true" CLASS=org.apache.hadoop.hdfs.server.mover.Mover hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
git commit: HDFS-6799. The invalidate method in SimulatedFSDataset failed to remove (invalidate) blocks from the file system. Contributed by Megasthenis Asteris.
Repository: hadoop Updated Branches: refs/heads/branch-2 5d897026e -> 215216abe HDFS-6799. The invalidate method in SimulatedFSDataset failed to remove (invalidate) blocks from the file system. Contributed by Megasthenis Asteris. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/215216ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/215216ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/215216ab Branch: refs/heads/branch-2 Commit: 215216abeb657de2d0ac8df52e7af5c4a04ba8cb Parents: 5d89702 Author: Benoy Antony Authored: Tue Sep 16 16:24:23 2014 -0700 Committer: Benoy Antony Committed: Tue Sep 16 16:27:25 2014 -0700 -- .../hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/215216ab/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 8ad4510..d0fad6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -482,9 +482,10 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override // FsDatasetSpi - public synchronized void unfinalizeBlock(ExtendedBlock b) { + public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException{ if (isValidRbw(b)) { - blockMap.remove(b.getLocalBlock()); + final Map map = getMap(b.getBlockPoolId()); + map.remove(b.getLocalBlock()); } } @@ -624,7 +625,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { continue; } storage.free(bpid, binfo.getNumBytes()); - blockMap.remove(b); + map.remove(b); } if (error) { throw new IOException("Invalidate: Missing blocks.");
git commit: HDFS-6799. The invalidate method in SimulatedFSDataset failed to remove (invalidate) blocks from the file system. Contributed by Megasthenis Asteris.
Repository: hadoop Updated Branches: refs/heads/trunk 7971c97ec -> 02adf7185 HDFS-6799. The invalidate method in SimulatedFSDataset failed to remove (invalidate) blocks from the file system. Contributed by Megasthenis Asteris. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02adf718 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02adf718 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02adf718 Branch: refs/heads/trunk Commit: 02adf7185de626492bba2b3718959457e958a7be Parents: 7971c97 Author: Benoy Antony Authored: Tue Sep 16 16:24:23 2014 -0700 Committer: Benoy Antony Committed: Tue Sep 16 16:24:23 2014 -0700 -- .../hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 7 --- 1 file changed, 4 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/02adf718/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 8ad4510..d0fad6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -482,9 +482,10 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override // FsDatasetSpi - public synchronized void unfinalizeBlock(ExtendedBlock b) { + public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException{ if (isValidRbw(b)) { - blockMap.remove(b.getLocalBlock()); + final Map map = getMap(b.getBlockPoolId()); + map.remove(b.getLocalBlock()); } } @@ -624,7 +625,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { continue; } storage.free(bpid, binfo.getNumBytes()); - blockMap.remove(b); + map.remove(b); } if (error) { throw new IOException("Invalidate: Missing blocks.");
git commit: HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/branch-2 7b9e76313 -> 39352a7a8 HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony) (cherry picked from commit 258c7d0f53fbdf4b0b9cae901701176e3e70c4fe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39352a7a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39352a7a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39352a7a Branch: refs/heads/branch-2 Commit: 39352a7a82eb5571494425dc1f78b579d9d03890 Parents: 7b9e763 Author: Benoy Antony Authored: Sat Aug 30 12:49:19 2014 -0700 Committer: Benoy Antony Committed: Sat Aug 30 13:08:04 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/security/alias/UserProvider.java | 5 - 2 files changed, 2 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/39352a7a/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 055a457..37ed2c7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -207,6 +207,8 @@ Release 2.6.0 - UNRELEASED schedules incoming calls and multiplexes outgoing calls. (Chris Li via Arpit Agarwal) +HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony) + BUG FIXES HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry http://git-wip-us.apache.org/repos/asf/hadoop/blob/39352a7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java index 9e724c0..e22c958 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java @@ -21,9 +21,7 @@ package org.apache.hadoop.security.alias; import java.io.IOException; import java.net.URI; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -41,8 +39,6 @@ public class UserProvider extends CredentialProvider { public static final String SCHEME_NAME = "user"; private final UserGroupInformation user; private final Credentials credentials; - private final Map cache = new HashMap(); private UserProvider() throws IOException { user = UserGroupInformation.getCurrentUser(); @@ -86,7 +82,6 @@ public class UserProvider extends CredentialProvider { throw new IOException("Credential " + name + " does not exist in " + this); } -cache.remove(name); } @Override
git commit: HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony)
Repository: hadoop Updated Branches: refs/heads/trunk 0f34e6f38 -> 258c7d0f5 HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/258c7d0f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/258c7d0f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/258c7d0f Branch: refs/heads/trunk Commit: 258c7d0f53fbdf4b0b9cae901701176e3e70c4fe Parents: 0f34e6f Author: Benoy Antony Authored: Sat Aug 30 12:49:19 2014 -0700 Committer: Benoy Antony Committed: Sat Aug 30 12:49:19 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/security/alias/UserProvider.java | 5 - 2 files changed, 2 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/258c7d0f/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3b44b8b..f3ef49d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -537,6 +537,8 @@ Release 2.6.0 - UNRELEASED schedules incoming calls and multiplexes outgoing calls. (Chris Li via Arpit Agarwal) +HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony) + BUG FIXES HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry http://git-wip-us.apache.org/repos/asf/hadoop/blob/258c7d0f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java index 99d6d00..262cbad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java @@ -21,9 +21,7 @@ package org.apache.hadoop.security.alias; import java.io.IOException; import java.net.URI; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -41,8 +39,6 @@ public class UserProvider extends CredentialProvider { public static final String SCHEME_NAME = "user"; private final UserGroupInformation user; private final Credentials credentials; - private final Map cache = new HashMap(); private UserProvider() throws IOException { user = UserGroupInformation.getCurrentUser(); @@ -86,7 +82,6 @@ public class UserProvider extends CredentialProvider { throw new IOException("Credential " + name + " does not exist in " + this); } -cache.remove(name); } @Override
svn commit: r1619959 - in /hadoop/common/site/main: author/src/documentation/content/xdocs/who.xml publish/who.html publish/who.pdf
Author: benoy Date: Sat Aug 23 03:45:59 2014 New Revision: 1619959 URL: http://svn.apache.org/r1619959 Log: Added benoy to committers list. Removed from PMC list. Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml hadoop/common/site/main/publish/who.html hadoop/common/site/main/publish/who.pdf Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml URL: http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1619959&r1=1619958&r2=1619959&view=diff == --- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml (original) +++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Sat Aug 23 03:45:59 2014 @@ -80,14 +80,6 @@ -benoy -http://people.apache.org/~benoy";>Benoy Antony -eBay - --8 - - - bikas Bikas Saha Hortonworks @@ -546,6 +538,14 @@ -8 + + + benoy + http://people.apache.org/~benoy";>Benoy Antony + eBay + + -8 + bikas Modified: hadoop/common/site/main/publish/who.html URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.html?rev=1619959&r1=1619958&r2=1619959&view=diff == --- hadoop/common/site/main/publish/who.html (original) +++ hadoop/common/site/main/publish/who.html Sat Aug 23 03:45:59 2014 @@ -341,17 +341,6 @@ document.write("Last Published: " + docu -benoy -http://people.apache.org/~benoy";>Benoy Antony -eBay - --8 - - - - - - bikas Bikas Saha Hortonworks @@ -860,7 +849,7 @@ document.write("Last Published: " + docu - + Emeritus Hadoop PMC Members @@ -875,7 +864,7 @@ document.write("Last Published: " + docu - + Hadoop Committers Hadoop's active committers include: @@ -979,6 +968,17 @@ document.write("Last Published: " + docu -8 + + + + +benoy + http://people.apache.org/~benoy";>Benoy Antony + eBay + + -8 + + Modified: hadoop/common/site/main/publish/who.pdf URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.pdf?rev=1619959&r1=1619958&r2=1619959&view=diff == Binary files - no diff available.
svn commit: r1619953 - in /hadoop/common/site/main: author/src/documentation/content/xdocs/who.xml publish/who.html publish/who.pdf
Author: benoy Date: Sat Aug 23 00:11:00 2014 New Revision: 1619953 URL: http://svn.apache.org/r1619953 Log: Added benoy to committers list. Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml hadoop/common/site/main/publish/who.html hadoop/common/site/main/publish/who.pdf Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml URL: http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1619953&r1=1619952&r2=1619953&view=diff == --- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml (original) +++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Sat Aug 23 00:11:00 2014 @@ -80,6 +80,14 @@ +benoy +http://people.apache.org/~benoy";>Benoy Antony +eBay + +-8 + + + bikas Bikas Saha Hortonworks Modified: hadoop/common/site/main/publish/who.html URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.html?rev=1619953&r1=1619952&r2=1619953&view=diff == --- hadoop/common/site/main/publish/who.html (original) +++ hadoop/common/site/main/publish/who.html Sat Aug 23 00:11:00 2014 @@ -341,6 +341,17 @@ document.write("Last Published: " + docu +benoy + http://people.apache.org/~benoy";>Benoy Antony +eBay + +-8 + + + + + + bikas Bikas Saha Hortonworks @@ -849,7 +860,7 @@ document.write("Last Published: " + docu - + Emeritus Hadoop PMC Members @@ -864,7 +875,7 @@ document.write("Last Published: " + docu - + Hadoop Committers Hadoop's active committers include: @@ -1755,7 +1766,7 @@ document.write("Last Published: " + docu - + Emeritus Hadoop Committers Hadoop committers who are no longer active include: Modified: hadoop/common/site/main/publish/who.pdf URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.pdf?rev=1619953&r1=1619952&r2=1619953&view=diff == Binary files - no diff available.