Author: tucu Date: Fri Aug 8 04:58:58 2014 New Revision: 1616672 URL: http://svn.apache.org/r1616672 Log: HADOOP-10771. Refactor HTTP delegation support out of httpfs to common, PART 2. (tucu)
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java?rev=1616672&r1=1616671&r2=1616672&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java Fri Aug 8 04:58:58 2014 @@ -39,12 +39,14 @@ import org.apache.hadoop.fs.permission.A import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.lib.wsrs.EnumSetParam; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; @@ -67,7 +69,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; -import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -75,7 +76,6 @@ import java.security.PrivilegedException import java.text.MessageFormat; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.Callable; /** * HttpFSServer implementation of the FileSystemAccess FileSystem. @@ -217,34 +217,15 @@ public class HttpFSFileSystem extends Fi } - - private AuthenticatedURL.Token authToken = new AuthenticatedURL.Token(); + private DelegationTokenAuthenticatedURL authURL; + private DelegationTokenAuthenticatedURL.Token authToken = + new DelegationTokenAuthenticatedURL.Token(); private URI uri; - private InetSocketAddress httpFSAddr; private Path workingDir; private UserGroupInformation realUser; private String doAs; - private Token<?> delegationToken; - //This method enables handling UGI doAs with SPNEGO, we have to - //fallback to the realuser who logged in with Kerberos credentials - private <T> T doAsRealUserIfNecessary(final Callable<T> callable) - throws IOException { - try { - if (realUser.getShortUserName().equals(doAs)) { - return callable.call(); - } else { - return realUser.doAs(new PrivilegedExceptionAction<T>() { - @Override - public T run() throws Exception { - return callable.call(); - } - }); - } - } catch (Exception ex) { - throw new IOException(ex.toString(), ex); - } - } + /** * Convenience method that creates a <code>HttpURLConnection</code> for the @@ -291,20 +272,26 @@ public class HttpFSFileSystem extends Fi private HttpURLConnection getConnection(final String method, Map<String, String> params, Map<String, List<String>> multiValuedParams, Path path, boolean makeQualified) throws IOException { - if (!realUser.getShortUserName().equals(doAs)) { - params.put(DO_AS_PARAM, doAs); - } - HttpFSKerberosAuthenticator.injectDelegationToken(params, delegationToken); if (makeQualified) { path = makeQualified(path); } final URL url = HttpFSUtils.createURL(path, params, multiValuedParams); - return doAsRealUserIfNecessary(new Callable<HttpURLConnection>() { - @Override - public HttpURLConnection call() throws Exception { - return getConnection(url, method); + try { + return UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction<HttpURLConnection>() { + @Override + public HttpURLConnection run() throws Exception { + return getConnection(url, method); + } + } + ); + } catch (Exception ex) { + if (ex instanceof IOException) { + throw (IOException) ex; + } else { + throw new IOException(ex); } - }); + } } /** @@ -321,12 +308,8 @@ public class HttpFSFileSystem extends Fi * @throws IOException thrown if an IO error occurrs. */ private HttpURLConnection getConnection(URL url, String method) throws IOException { - Class<? extends Authenticator> klass = - getConf().getClass("httpfs.authenticator.class", - HttpFSKerberosAuthenticator.class, Authenticator.class); - Authenticator authenticator = ReflectionUtils.newInstance(klass, getConf()); try { - HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(url, authToken); + HttpURLConnection conn = authURL.openConnection(url, authToken); conn.setRequestMethod(method); if (method.equals(HTTP_POST) || method.equals(HTTP_PUT)) { conn.setDoOutput(true); @@ -357,10 +340,17 @@ public class HttpFSFileSystem extends Fi super.initialize(name, conf); try { uri = new URI(name.getScheme() + "://" + name.getAuthority()); - httpFSAddr = NetUtils.createSocketAddr(getCanonicalUri().toString()); } catch (URISyntaxException ex) { throw new IOException(ex); } + + Class<? extends DelegationTokenAuthenticator> klass = + getConf().getClass("httpfs.authenticator.class", + KerberosDelegationTokenAuthenticator.class, + DelegationTokenAuthenticator.class); + DelegationTokenAuthenticator authenticator = + ReflectionUtils.newInstance(klass, getConf()); + authURL = new DelegationTokenAuthenticatedURL(authenticator); } @Override @@ -1059,38 +1049,57 @@ public class HttpFSFileSystem extends Fi @Override public Token<?> getDelegationToken(final String renewer) throws IOException { - return doAsRealUserIfNecessary(new Callable<Token<?>>() { - @Override - public Token<?> call() throws Exception { - return HttpFSKerberosAuthenticator. - getDelegationToken(uri, httpFSAddr, authToken, renewer); + try { + return UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction<Token<?>>() { + @Override + public Token<?> run() throws Exception { + return authURL.getDelegationToken(uri.toURL(), authToken, + renewer); + } + } + ); + } catch (Exception ex) { + if (ex instanceof IOException) { + throw (IOException) ex; + } else { + throw new IOException(ex); } - }); + } } public long renewDelegationToken(final Token<?> token) throws IOException { - return doAsRealUserIfNecessary(new Callable<Long>() { - @Override - public Long call() throws Exception { - return HttpFSKerberosAuthenticator. - renewDelegationToken(uri, authToken, token); + try { + return UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction<Long>() { + @Override + public Long run() throws Exception { + return authURL.renewDelegationToken(uri.toURL(), authToken); + } + } + ); + } catch (Exception ex) { + if (ex instanceof IOException) { + throw (IOException) ex; + } else { + throw new IOException(ex); } - }); + } } public void cancelDelegationToken(final Token<?> token) throws IOException { - HttpFSKerberosAuthenticator. - cancelDelegationToken(uri, authToken, token); + authURL.cancelDelegationToken(uri.toURL(), authToken); } @Override public Token<?> getRenewToken() { - return delegationToken; + return null; //TODO : for renewer } @Override + @SuppressWarnings("unchecked") public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) { - delegationToken = token; + //TODO : for renewer } @Override Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java?rev=1616672&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java (added) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSAuthenticationFilter.java Fri Aug 8 04:58:58 2014 @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.http.server; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.util.Map; +import java.util.Properties; + +/** + * Subclass of hadoop-auth <code>AuthenticationFilter</code> that obtains its configuration + * from HttpFSServer's server configuration. + */ +@InterfaceAudience.Private +public class HttpFSAuthenticationFilter + extends DelegationTokenAuthenticationFilter { + + private static final String CONF_PREFIX = "httpfs.authentication."; + + private static final String SIGNATURE_SECRET_FILE = SIGNATURE_SECRET + ".file"; + + /** + * Returns the hadoop-auth configuration from HttpFSServer's configuration. + * <p/> + * It returns all HttpFSServer's configuration properties prefixed with + * <code>httpfs.authentication</code>. The <code>httpfs.authentication</code> + * prefix is removed from the returned property names. + * + * @param configPrefix parameter not used. + * @param filterConfig parameter not used. + * + * @return hadoop-auth configuration read from HttpFSServer's configuration. + */ + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException{ + Properties props = new Properties(); + Configuration conf = HttpFSServerWebApp.get().getConfig(); + + props.setProperty(AuthenticationFilter.COOKIE_PATH, "/"); + for (Map.Entry<String, String> entry : conf) { + String name = entry.getKey(); + if (name.startsWith(CONF_PREFIX)) { + String value = conf.get(name); + name = name.substring(CONF_PREFIX.length()); + props.setProperty(name, value); + } + } + + String signatureSecretFile = props.getProperty(SIGNATURE_SECRET_FILE, null); + if (signatureSecretFile == null) { + throw new RuntimeException("Undefined property: " + SIGNATURE_SECRET_FILE); + } + + try { + StringBuilder secret = new StringBuilder(); + Reader reader = new FileReader(signatureSecretFile); + int c = reader.read(); + while (c > -1) { + secret.append((char)c); + c = reader.read(); + } + reader.close(); + props.setProperty(AuthenticationFilter.SIGNATURE_SECRET, secret.toString()); + } catch (IOException ex) { + throw new RuntimeException("Could not read HttpFS signature secret file: " + signatureSecretFile); + } + return props; + } + +} Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml?rev=1616672&r1=1616671&r2=1616672&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/resources/httpfs-default.xml Fri Aug 8 04:58:58 2014 @@ -35,7 +35,6 @@ org.apache.hadoop.lib.service.scheduler.SchedulerService, org.apache.hadoop.lib.service.security.GroupsService, org.apache.hadoop.lib.service.security.ProxyUserService, - org.apache.hadoop.lib.service.security.DelegationTokenManagerService, org.apache.hadoop.lib.service.hadoop.FileSystemAccessService </value> <description> @@ -226,12 +225,4 @@ </description> </property> - <property> - <name>httpfs.user.provider.user.pattern</name> - <value>^[A-Za-z_][A-Za-z0-9._-]*[$]?$</value> - <description> - Valid pattern for user and group names, it must be a valid java regex. - </description> - </property> - </configuration> Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java?rev=1616672&r1=1616671&r2=1616672&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandlerForTesting.java Fri Aug 8 04:58:58 2014 @@ -17,15 +17,19 @@ */ package org.apache.hadoop.fs.http.server; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; + import javax.servlet.ServletException; import java.util.Properties; public class HttpFSKerberosAuthenticationHandlerForTesting - extends HttpFSKerberosAuthenticationHandler { + extends KerberosDelegationTokenAuthenticationHandler { @Override public void init(Properties config) throws ServletException { //NOP overwrite to avoid Kerberos initialization + config.setProperty(TOKEN_KIND, "t"); + initTokenManager(config); } @Override Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java?rev=1616672&r1=1616671&r2=1616672&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java Fri Aug 8 04:58:58 2014 @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.http.server; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; import org.json.simple.JSONArray; import org.junit.Assert; @@ -43,7 +45,6 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrCodec; -import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; import org.apache.hadoop.lib.server.Service; import org.apache.hadoop.lib.server.ServiceException; import org.apache.hadoop.lib.service.Groups; @@ -682,7 +683,7 @@ public class TestHttpFSServer extends HF AuthenticationToken token = new AuthenticationToken("u", "p", - HttpFSKerberosAuthenticationHandlerForTesting.TYPE); + new KerberosDelegationTokenAuthenticationHandler().getType()); token.setExpires(System.currentTimeMillis() + 100000000); Signer signer = new Signer(new StringSignerSecretProvider("secret")); String tokenSigned = signer.sign(token.toString()); @@ -706,9 +707,9 @@ public class TestHttpFSServer extends HF JSONObject json = (JSONObject) new JSONParser().parse(new InputStreamReader(conn.getInputStream())); json = (JSONObject) - json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON); + json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON); String tokenStr = (String) - json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); + json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); url = new URL(TestJettyHelper.getJettyURL(), "/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java?rev=1616672&r1=1616671&r2=1616672&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java Fri Aug 8 04:58:58 2014 @@ -23,11 +23,11 @@ import org.apache.hadoop.fs.DelegationTo import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.http.client.HttpFSFileSystem; -import org.apache.hadoop.fs.http.client.HttpFSKerberosAuthenticator; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; import org.apache.hadoop.test.HFSTestCase; import org.apache.hadoop.test.KerberosTestUtils; import org.apache.hadoop.test.TestDir; @@ -166,9 +166,9 @@ public class TestHttpFSWithKerberos exte .parse(new InputStreamReader(conn.getInputStream())); json = (JSONObject) json - .get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON); + .get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON); String tokenStr = (String) json - .get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); + .get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); //access httpfs using the delegation token url = new URL(TestJettyHelper.getJettyURL(),