Author: szetszwo Date: Fri Feb 28 19:37:47 2014 New Revision: 1573043 URL: http://svn.apache.org/r1573043 Log: Merge r1569890 through r1573042 from trunk.
Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HsftpFileSystem.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpDelegationToken.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1572723-1573042 Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Feb 28 19:37:47 2014 @@ -367,6 +367,9 @@ Release 2.4.0 - UNRELEASED HDFS-6025. Update findbugsExcludeFile.xml. (szetszwo) + HDFS-6030. Remove an unused constructor in INode.java. (yzhang via + cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery @@ -499,6 +502,9 @@ Release 2.4.0 - UNRELEASED HDFS-5936. MiniDFSCluster does not clean data left behind by SecondaryNameNode. (Binglin Chang via cnauroth) + HDFS-5339. WebHDFS URI does not accept logical nameservices when security is + enabled. (Haohui Mai via jing9) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) @@ -642,6 +648,9 @@ Release 2.4.0 - UNRELEASED HDFS-5908. Change AclFeature to capture list of ACL entries in an ImmutableList. (cnauroth) + HDFS-6028. Print clearer error message when user attempts to delete required + mask entry from ACL. (cnauroth) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1572418-1573042 Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java Fri Feb 28 19:37:47 2014 @@ -365,7 +365,7 @@ final class AclTransformation { maskDirty.contains(scope)) { // Caller explicitly removed mask entry, but it's required. throw new AclException( - "Invalid ACL: mask is required, but it was deleted."); + "Invalid ACL: mask is required and cannot be deleted."); } else if (providedMask.containsKey(scope) && (!scopeDirty.contains(scope) || maskDirty.contains(scope))) { // Caller explicitly provided new mask, or we are preserving the existing Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java Fri Feb 28 19:37:47 2014 @@ -752,11 +752,6 @@ public abstract class INode implements I */ private List<Block> toDeleteList; - public BlocksMapUpdateInfo(List<Block> toDeleteList) { - this.toDeleteList = toDeleteList == null ? new ArrayList<Block>() - : toDeleteList; - } - public BlocksMapUpdateInfo() { toDeleteList = new ChunkedArrayList<Block>(); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java Fri Feb 28 19:37:47 2014 @@ -97,9 +97,10 @@ public class HftpFileSystem extends File public static final String HFTP_TIMEZONE = "UTC"; public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ"; - protected TokenAspect<HftpFileSystem> tokenAspect; + protected TokenAspect<? extends HftpFileSystem> tokenAspect; private Token<?> delegationToken; private Token<?> renewToken; + protected Text tokenServiceName; @Override public URI getCanonicalUri() { @@ -175,9 +176,8 @@ public class HftpFileSystem extends File * Initialize connectionFactory and tokenAspect. This function is intended to * be overridden by HsFtpFileSystem. */ - protected void initTokenAspect(Configuration conf) - throws IOException { - tokenAspect = new TokenAspect<HftpFileSystem>(this, TOKEN_KIND); + protected void initTokenAspect() { + tokenAspect = new TokenAspect<HftpFileSystem>(this, tokenServiceName, TOKEN_KIND); } @Override @@ -189,6 +189,7 @@ public class HftpFileSystem extends File .newDefaultURLConnectionFactory(conf); this.ugi = UserGroupInformation.getCurrentUser(); this.nnUri = getNamenodeUri(name); + this.tokenServiceName = SecurityUtil.buildTokenService(nnUri); try { this.hftpURI = new URI(name.getScheme(), name.getAuthority(), @@ -197,7 +198,7 @@ public class HftpFileSystem extends File throw new IllegalArgumentException(e); } - initTokenAspect(conf); + initTokenAspect(); if (UserGroupInformation.isSecurityEnabled()) { tokenAspect.initDelegationToken(ugi); } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HsftpFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HsftpFileSystem.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HsftpFileSystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HsftpFileSystem.java Fri Feb 28 19:37:47 2014 @@ -18,11 +18,8 @@ package org.apache.hadoop.hdfs.web; -import java.io.IOException; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.io.Text; @@ -60,8 +57,9 @@ public class HsftpFileSystem extends Hft } @Override - protected void initTokenAspect(Configuration conf) throws IOException { - tokenAspect = new TokenAspect<HftpFileSystem>(this, TOKEN_KIND); + protected void initTokenAspect() { + tokenAspect = new TokenAspect<HsftpFileSystem>(this, tokenServiceName, + TOKEN_KIND); } @Override Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java Fri Feb 28 19:37:47 2014 @@ -37,7 +37,7 @@ public class SWebHdfsFileSystem extends @Override protected synchronized void initializeTokenAspect() { - tokenAspect = new TokenAspect<WebHdfsFileSystem>(this, TOKEN_KIND); + tokenAspect = new TokenAspect<SWebHdfsFileSystem>(this, tokenServiceName, TOKEN_KIND); } @Override Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/TokenAspect.java Fri Feb 28 19:37:47 2014 @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.web; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.util.Collection; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,7 +30,6 @@ import org.apache.hadoop.fs.DelegationTo import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -93,23 +91,11 @@ final class TokenAspect<T extends FileSy } } - static class DTSelecorByKind extends + private static class DTSelecorByKind extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> { - private static final DelegationTokenSelector selector = new DelegationTokenSelector(); - public DTSelecorByKind(final Text kind) { super(kind); } - - Token<DelegationTokenIdentifier> selectToken(URI nnUri, - Collection<Token<?>> tokens, Configuration conf) { - Token<DelegationTokenIdentifier> token = selectToken( - SecurityUtil.buildTokenService(nnUri), tokens); - if (token == null) { - token = selector.selectToken(nnUri, tokens, conf); - } - return token; - } } /** @@ -117,9 +103,6 @@ final class TokenAspect<T extends FileSy */ interface TokenManagementDelegator { void cancelDelegationToken(final Token<?> token) throws IOException; - - URI getCanonicalUri(); - long renewDelegationToken(final Token<?> token) throws IOException; } @@ -129,11 +112,13 @@ final class TokenAspect<T extends FileSy private final T fs; private boolean hasInitedToken; private final Log LOG; + private final Text serviceName; - TokenAspect(T fs, final Text kind) { + TokenAspect(T fs, final Text serviceName, final Text kind) { this.LOG = LogFactory.getLog(fs.getClass()); this.fs = fs; this.dtSelector = new DTSelecorByKind(kind); + this.serviceName = serviceName; } synchronized void ensureTokenInitialized() throws IOException { @@ -173,9 +158,7 @@ final class TokenAspect<T extends FileSy @VisibleForTesting Token<DelegationTokenIdentifier> selectDelegationToken( UserGroupInformation ugi) { - return dtSelector.selectToken( - ((TokenManagementDelegator)fs).getCanonicalUri(), ugi.getTokens(), - fs.getConf()); + return dtSelector.selectToken(serviceName, ugi.getTokens()); } private synchronized void addRenewAction(final T webhdfs) { Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Fri Feb 28 19:37:47 2014 @@ -27,7 +27,6 @@ import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; import java.security.PrivilegedExceptionAction; import java.util.List; @@ -119,11 +118,12 @@ public class WebHdfsFileSystem extends F /** Delegation token kind */ public static final Text TOKEN_KIND = new Text("WEBHDFS delegation"); - protected TokenAspect<WebHdfsFileSystem> tokenAspect; + protected TokenAspect<? extends WebHdfsFileSystem> tokenAspect; private UserGroupInformation ugi; private URI uri; private Token<?> delegationToken; + protected Text tokenServiceName; private RetryPolicy retryPolicy = null; private Path workingDir; private InetSocketAddress nnAddrs[]; @@ -152,7 +152,8 @@ public class WebHdfsFileSystem extends F * be overridden by SWebHdfsFileSystem. */ protected synchronized void initializeTokenAspect() { - tokenAspect = new TokenAspect<WebHdfsFileSystem>(this, TOKEN_KIND); + tokenAspect = new TokenAspect<WebHdfsFileSystem>(this, tokenServiceName, + TOKEN_KIND); } @Override @@ -161,23 +162,26 @@ public class WebHdfsFileSystem extends F super.initialize(uri, conf); setConf(conf); /** set user pattern based on configuration file */ - UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); + UserParam.setUserPattern(conf.get( + DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, + DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); + connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); - initializeTokenAspect(); ugi = UserGroupInformation.getCurrentUser(); + this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); + this.nnAddrs = DFSUtil.resolveWebHdfsUri(this.uri, conf); - try { - this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, - null, null); - this.nnAddrs = DFSUtil.resolveWebHdfsUri(this.uri, conf); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } + boolean isHA = HAUtil.isLogicalUri(conf, this.uri); + // In non-HA case, the code needs to call getCanonicalUri() in order to + // handle the case where no port is specified in the URI + this.tokenServiceName = isHA ? HAUtil.buildTokenServiceForLogicalUri(uri) + : SecurityUtil.buildTokenService(getCanonicalUri()); + initializeTokenAspect(); - if (!HAUtil.isLogicalUri(conf, this.uri)) { + if (!isHA) { this.retryPolicy = RetryUtils.getDefaultRetryPolicy( conf, @@ -1003,20 +1007,20 @@ public class WebHdfsFileSystem extends F final String renewer) throws IOException { final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN; final Map<?, ?> m = run(op, null, new RenewerParam(renewer)); - final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m); - SecurityUtil.setTokenService(token, getCurrentNNAddr()); + final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m); + token.setService(tokenServiceName); return token; } @Override - public Token<?> getRenewToken() { + public synchronized Token<?> getRenewToken() { return delegationToken; } @Override public <T extends TokenIdentifier> void setDelegationToken( final Token<T> token) { - synchronized(this) { + synchronized (this) { delegationToken = token; } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpDelegationToken.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpDelegationToken.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpDelegationToken.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHftpDelegationToken.java Fri Feb 28 19:37:47 2014 @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.token.Token; @@ -73,14 +74,15 @@ public class TestHftpDelegationToken { doReturn(conn).when(factory).openConnection(Mockito.<URL> any(), anyBoolean()); - fs.initialize(new URI("hftp://127.0.0.1:8020"), conf); + final URI uri = new URI("hftp://127.0.0.1:8020"); + fs.initialize(uri, conf); fs.connectionFactory = factory; UserGroupInformation ugi = UserGroupInformation.createUserForTesting("foo", new String[] { "bar" }); TokenAspect<HftpFileSystem> tokenAspect = new TokenAspect<HftpFileSystem>( - fs, HftpFileSystem.TOKEN_KIND); + fs, SecurityUtil.buildTokenService(uri), HftpFileSystem.TOKEN_KIND); tokenAspect.initDelegationToken(ugi); tokenAspect.ensureTokenInitialized(); Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestTokenAspect.java Fri Feb 28 19:37:47 2014 @@ -48,9 +48,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.SecurityUtilTestHelper; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -122,14 +121,9 @@ public class TestTokenAspect { public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); setConf(conf); - try { - this.uri = new URI(name.getScheme(), name.getAuthority(), null, null, - null); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - - tokenAspect = new TokenAspect<DummyFs>(this, DummyFs.TOKEN_KIND); + this.uri = URI.create(name.getScheme() + "://" + name.getAuthority()); + tokenAspect = new TokenAspect<DummyFs>(this, + SecurityUtil.buildTokenService(uri), TOKEN_KIND); if (emulateSecurityEnabled || UserGroupInformation.isSecurityEnabled()) { tokenAspect.initDelegationToken(ugi); } @@ -293,9 +287,10 @@ public class TestTokenAspect { doThrow(new IOException("get failed")).when(fs).addDelegationTokens(null, null); + final URI uri = new URI("dummyfs://127.0.0.1:1234"); TokenAspect<DummyFs> tokenAspect = new TokenAspect<DummyFs>(fs, - DummyFs.TOKEN_KIND); - fs.initialize(new URI("dummyfs://127.0.0.1:1234"), conf); + SecurityUtil.buildTokenService(uri), DummyFs.TOKEN_KIND); + fs.initialize(uri, conf); tokenAspect.initDelegationToken(ugi); // trigger token acquisition @@ -318,58 +313,4 @@ public class TestTokenAspect { action = getActionFromTokenAspect(tokenAspect); assertTrue(action.isValid()); } - - @Test - public void testTokenSelectionPreferences() throws IOException, - URISyntaxException { - Configuration conf = new Configuration(); - DummyFs fs = spy(new DummyFs()); - doReturn(null).when(fs).getDelegationToken(anyString()); - fs.initialize(new URI("dummyfs://localhost:1234"), conf); - TokenAspect<DummyFs> aspect = new TokenAspect<DummyFs>(fs, - DummyFs.TOKEN_KIND); - UserGroupInformation ugi = UserGroupInformation.createUserForTesting("foo", - new String[] { "bar" }); - - // use ip-based tokens - SecurityUtilTestHelper.setTokenServiceUseIp(true); - - // test fallback to hdfs token - Token<TokenIdentifier> hdfsToken = new Token<TokenIdentifier>(new byte[0], - new byte[0], DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text( - "127.0.0.1:8020")); - ugi.addToken(hdfsToken); - - // test fallback to hdfs token - Token<?> token = aspect.selectDelegationToken(ugi); - assertEquals(hdfsToken, token); - - // test dummyfs is favored over hdfs - Token<TokenIdentifier> dummyFsToken = new Token<TokenIdentifier>( - new byte[0], new byte[0], DummyFs.TOKEN_KIND, - new Text("127.0.0.1:1234")); - ugi.addToken(dummyFsToken); - token = aspect.selectDelegationToken(ugi); - assertEquals(dummyFsToken, token); - - // switch to using host-based tokens, no token should match - SecurityUtilTestHelper.setTokenServiceUseIp(false); - token = aspect.selectDelegationToken(ugi); - assertNull(token); - - // test fallback to hdfs token - hdfsToken = new Token<TokenIdentifier>(new byte[0], new byte[0], - DelegationTokenIdentifier.HDFS_DELEGATION_KIND, new Text( - "localhost:8020")); - ugi.addToken(hdfsToken); - token = aspect.selectDelegationToken(ugi); - assertEquals(hdfsToken, token); - - // test dummyfs is favored over hdfs - dummyFsToken = new Token<TokenIdentifier>(new byte[0], new byte[0], - DummyFs.TOKEN_KIND, new Text("localhost:1234")); - ugi.addToken(dummyFsToken); - token = aspect.selectDelegationToken(ugi); - assertEquals(dummyFsToken, token); - } } Modified: hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java?rev=1573043&r1=1573042&r2=1573043&view=diff ============================================================================== --- hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java (original) +++ hadoop/common/branches/HDFS-5535/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java Fri Feb 28 19:37:47 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.web; +import java.io.IOException; import java.net.URI; import org.apache.hadoop.conf.Configuration; @@ -27,35 +28,34 @@ import org.apache.hadoop.hdfs.DFSConfigK import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import org.apache.hadoop.security.token.Token; import org.junit.Assert; import org.junit.Test; /** Test whether WebHDFS can connect to an HA cluster */ public class TestWebHDFSForHA { - private static final String LOGICAL_NAME = "minidfs"; + private static final MiniDFSNNTopology topo = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf(LOGICAL_NAME).addNN( + new MiniDFSNNTopology.NNConf("nn1")).addNN( + new MiniDFSNNTopology.NNConf("nn2"))); @Test - public void test() throws Exception { + public void testHA() throws IOException { Configuration conf = new Configuration(); - conf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); - - MiniDFSNNTopology topo = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf(LOGICAL_NAME).addNN( - new MiniDFSNNTopology.NNConf("nn1")).addNN( - new MiniDFSNNTopology.NNConf("nn2"))); - - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) - .numDataNodes(3).build(); - - HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); - + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); + MiniDFSCluster cluster = null; FileSystem fs = null; try { + cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) + .numDataNodes(0).build(); + + HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); + cluster.waitActive(); final String uri = WebHdfsFileSystem.SCHEME + "://" + LOGICAL_NAME; - fs = (WebHdfsFileSystem) FileSystem.get(new URI(uri), conf); + fs = FileSystem.get(URI.create(uri), conf); cluster.transitionToActive(0); final Path dir = new Path("/test"); @@ -66,12 +66,50 @@ public class TestWebHDFSForHA { final Path dir2 = new Path("/test2"); Assert.assertTrue(fs.mkdirs(dir2)); + } finally { + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + @Test + public void testSecureHA() throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, + true); + conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); + + MiniDFSCluster cluster = null; + WebHdfsFileSystem fs = null; + try { + cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) + .numDataNodes(0).build(); + + HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); + cluster.waitActive(); + + final String uri = WebHdfsFileSystem.SCHEME + "://" + LOGICAL_NAME; + fs = (WebHdfsFileSystem) FileSystem.get(URI.create(uri), conf); + + cluster.transitionToActive(0); + Token<?> token = fs.getDelegationToken(null); + + cluster.shutdownNameNode(0); + cluster.transitionToActive(1); + + fs.renewDelegationToken(token); + fs.cancelDelegationToken(token); } finally { if (fs != null) { fs.close(); } - cluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); + } } } -} +} \ No newline at end of file