Author: suresh Date: Tue Jan 24 01:24:53 2012 New Revision: 1235107 URL: http://svn.apache.org/viewvc?rev=1235107&view=rev Log: HADOOP-7964. Deadlock in NetUtils and SecurityUtil class initialization. Contributed by Daryn Sharp.
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/NetUtilsTestResolver.java - copied, changed from r1235092, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/NetUtilsTestResolver.java Removed: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/NetUtilsTestResolver.java Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/core/org/apache/hadoop/net/NetUtils.java hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileSystem.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/TestNetUtils.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1235107&r1=1235106&r2=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Tue Jan 24 01:24:53 2012 @@ -89,6 +89,9 @@ Release 1.1.0 - unreleased HADOOP-7982. UserGroupInformation fails to login if thread's context classloader can't load HadoopLoginModule. (todd) + HADOOP-7964. Deadlock in NetUtils and SecurityUtil class initialization. + (Daryn Sharp via suresh) + IMPROVEMENTS MAPREDUCE-2517. Add system tests to Gridmix. (Vinay Thota via amarrk) Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/net/NetUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/net/NetUtils.java?rev=1235107&r1=1235106&r2=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/net/NetUtils.java (original) +++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/net/NetUtils.java Tue Jan 24 01:24:53 2012 @@ -45,35 +45,12 @@ import org.apache.hadoop.ipc.VersionedPr import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.ReflectionUtils; -// this will need to be replaced someday when there is a suitable replacement -import sun.net.dns.ResolverConfiguration; -import sun.net.util.IPAddressUtil; - public class NetUtils { private static final Log LOG = LogFactory.getLog(NetUtils.class); private static Map<String, String> hostToResolved = new HashMap<String, String>(); - private static HostResolver hostResolver; - - static { - // SecurityUtils requires a more secure host resolver if tokens are - // using hostnames - setUseQualifiedHostResolver(!SecurityUtil.getTokenServiceUseIp()); - } - - /** - * This method is intended for use only by SecurityUtils! - * @param flag where the qualified or standard host resolver is used - * to create socket addresses - */ - public static void setUseQualifiedHostResolver(boolean flag) { - hostResolver = flag - ? new QualifiedHostResolver() - : new StandardHostResolver(); - } - /** * Get the socket factory for the given class according to its * configuration parameter @@ -206,7 +183,7 @@ public class NetUtils { InetSocketAddress addr; try { - InetAddress iaddr = hostResolver.getByName(resolveHost); + InetAddress iaddr = SecurityUtil.getByName(resolveHost); // if there is a static entry for the host, make the returned // address look like the original given host if (staticHost != null) { @@ -219,150 +196,6 @@ public class NetUtils { return addr; } - protected interface HostResolver { - InetAddress getByName(String host) throws UnknownHostException; - } - - /** - * Uses standard java host resolution - */ - protected static class StandardHostResolver implements HostResolver { - public InetAddress getByName(String host) throws UnknownHostException { - return InetAddress.getByName(host); - } - } - - /** - * This an alternate resolver with important properties that the standard - * java resolver lacks: - * 1) The hostname is fully qualified. This avoids security issues if not - * all hosts in the cluster do not share the same search domains. It - * also prevents other hosts from performing unnecessary dns searches. - * In contrast, InetAddress simply returns the host as given. - * 2) The InetAddress is instantiated with an exact host and IP to prevent - * further unnecessary lookups. InetAddress may perform an unnecessary - * reverse lookup for an IP. - * 3) A call to getHostName() will always return the qualified hostname, or - * more importantly, the IP if instantiated with an IP. This avoids - * unnecessary dns timeouts if the host is not resolvable. - * 4) Point 3 also ensures that if the host is re-resolved, ex. during a - * connection re-attempt, that a reverse lookup to host and forward - * lookup to IP is not performed since the reverse/forward mappings may - * not always return the same IP. If the client initiated a connection - * with an IP, then that IP is all that should ever be contacted. - * - * NOTE: this resolver is only used if: - * hadoop.security.token.service.use_ip=false - */ - protected static class QualifiedHostResolver implements HostResolver { - @SuppressWarnings("unchecked") - private List<String> searchDomains = - ResolverConfiguration.open().searchlist(); - - /** - * Create an InetAddress with a fully qualified hostname of the given - * hostname. InetAddress does not qualify an incomplete hostname that - * is resolved via the domain search list. - * {@link InetAddress#getCanonicalHostName()} will fully qualify the - * hostname, but it always return the A record whereas the given hostname - * may be a CNAME. - * - * @param host a hostname or ip address - * @return InetAddress with the fully qualified hostname or ip - * @throws UnknownHostException if host does not exist - */ - public InetAddress getByName(String host) throws UnknownHostException { - InetAddress addr = null; - - if (IPAddressUtil.isIPv4LiteralAddress(host)) { - // use ipv4 address as-is - byte[] ip = IPAddressUtil.textToNumericFormatV4(host); - addr = InetAddress.getByAddress(host, ip); - } else if (IPAddressUtil.isIPv6LiteralAddress(host)) { - // use ipv6 address as-is - byte[] ip = IPAddressUtil.textToNumericFormatV6(host); - addr = InetAddress.getByAddress(host, ip); - } else if (host.endsWith(".")) { - // a rooted host ends with a dot, ex. "host." - // rooted hosts never use the search path, so only try an exact lookup - addr = getByExactName(host); - } else if (host.contains(".")) { - // the host contains a dot (domain), ex. "host.domain" - // try an exact host lookup, then fallback to search list - addr = getByExactName(host); - if (addr == null) { - addr = getByNameWithSearch(host); - } - } else { - // it's a simple host with no dots, ex. "host" - // try the search list, then fallback to exact host - InetAddress loopback = InetAddress.getByName(null); - if (host.equalsIgnoreCase(loopback.getHostName())) { - addr = InetAddress.getByAddress(host, loopback.getAddress()); - } else { - addr = getByNameWithSearch(host); - if (addr == null) { - addr = getByExactName(host); - } - } - } - // unresolvable! - if (addr == null) { - throw new UnknownHostException(host); - } - return addr; - } - - InetAddress getByExactName(String host) { - InetAddress addr = null; - // InetAddress will use the search list unless the host is rooted - // with a trailing dot. The trailing dot will disable any use of the - // search path in a lower level resolver. See RFC 1535. - String fqHost = host; - if (!fqHost.endsWith(".")) fqHost += "."; - try { - addr = getInetAddressByName(fqHost); - // can't leave the hostname as rooted or other parts of the system - // malfunction, ex. kerberos principals are lacking proper host - // equivalence for rooted/non-rooted hostnames - addr = InetAddress.getByAddress(host, addr.getAddress()); - } catch (UnknownHostException e) { - // ignore, caller will throw if necessary - } - return addr; - } - - InetAddress getByNameWithSearch(String host) { - InetAddress addr = null; - if (host.endsWith(".")) { // already qualified? - addr = getByExactName(host); - } else { - for (String domain : searchDomains) { - String dot = !domain.startsWith(".") ? "." : ""; - addr = getByExactName(host + dot + domain); - if (addr != null) break; - } - } - return addr; - } - - // implemented as a separate method to facilitate unit testing - InetAddress getInetAddressByName(String host) throws UnknownHostException { - return InetAddress.getByName(host); - } - - void setSearchDomains(String ... domains) { - searchDomains = Arrays.asList(domains); - } - } - - /** - * This is for testing only! - */ - static void setHostResolver(HostResolver newResolver) { - hostResolver = newResolver; - } - /** * Resolve the uri's hostname and add the default port if not in the uri * @param uri to resolve @@ -404,7 +237,7 @@ public class NetUtils { String fqHost = canonicalizedHostCache.get(host); if (fqHost == null) { try { - fqHost = hostResolver.getByName(host).getHostName(); + fqHost = SecurityUtil.getByName(host).getHostName(); // slight race condition, but won't hurt canonicalizedHostCache.put(host, fqHost); } catch (UnknownHostException e) { Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java?rev=1235107&r1=1235106&r2=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java (original) +++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/security/SecurityUtil.java Tue Jan 24 01:24:53 2012 @@ -23,6 +23,8 @@ import java.net.URI; import java.net.URL; import java.net.UnknownHostException; import java.security.AccessController; +import java.util.Arrays; +import java.util.List; import java.util.Set; import javax.security.auth.Subject; @@ -37,6 +39,9 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; +//this will need to be replaced someday when there is a suitable replacement +import sun.net.dns.ResolverConfiguration; +import sun.net.util.IPAddressUtil; import sun.security.jgss.krb5.Krb5Util; import sun.security.krb5.Credentials; import sun.security.krb5.PrincipalName; @@ -46,8 +51,9 @@ public class SecurityUtil { public static final String HOSTNAME_PATTERN = "_HOST"; // controls whether buildTokenService will use an ip or host/ip as given - // by the user - private static boolean useIpForTokenService; + // by the user; visible for testing + static boolean useIpForTokenService; + static HostResolver hostResolver; static { boolean useIp = new Configuration().getBoolean( @@ -61,15 +67,9 @@ public class SecurityUtil { */ static void setTokenServiceUseIp(boolean flag) { useIpForTokenService = flag; - NetUtils.setUseQualifiedHostResolver(!flag); - } - - /** - * Intended only for temporary use by NetUtils. Do not use. - * @return whether tokens use an IP address - */ - public static boolean getTokenServiceUseIp() { - return useIpForTokenService; + hostResolver = !useIpForTokenService + ? new QualifiedHostResolver() + : new StandardHostResolver(); } /** @@ -119,6 +119,7 @@ public class SecurityUtil { * it will be removed when the Java behavior is changed. * * @param remoteHost Target URL the krb-https client will access + * @throws IOException if a service ticket is not available */ public static void fetchServiceTicket(URL remoteHost) throws IOException { if(!UserGroupInformation.isSecurityEnabled()) @@ -155,7 +156,7 @@ public class SecurityUtil { * @param hostname * the fully-qualified domain name used for substitution * @return converted Kerberos principal name - * @throws IOException + * @throws IOException if the service ticket cannot be retrieved */ public static String getServerPrincipal(String principalConfig, String hostname) throws IOException { @@ -180,7 +181,7 @@ public class SecurityUtil { * @param addr * InetAddress of the host used for substitution * @return converted Kerberos principal name - * @throws IOException + * @throws IOException if the client address cannot be determined */ public static String getServerPrincipal(String principalConfig, InetAddress addr) throws IOException { @@ -227,7 +228,7 @@ public class SecurityUtil { * the key to look for keytab file in conf * @param userNameKey * the key to look for user's Kerberos principal name in conf - * @throws IOException + * @throws IOException if the client address cannot be determined */ public static void login(final Configuration conf, final String keytabFileKey, final String userNameKey) throws IOException { @@ -246,7 +247,7 @@ public class SecurityUtil { * the key to look for user's Kerberos principal name in conf * @param hostname * hostname to use for substitution - * @throws IOException + * @throws IOException if login fails */ public static void login(final Configuration conf, final String keytabFileKey, final String userNameKey, String hostname) @@ -345,4 +346,155 @@ public class SecurityUtil { public static String getHostFromPrincipal(String principalName) { return new KerberosName(principalName).getHostName(); } + + /** + * Resolves a host subject to the security requirements determined by + * hadoop.security.token.service.use_ip. + * + * @param hostname host or ip to resolve + * @return a resolved host + * @throws UnknownHostException if the host doesn't exist + */ + //@InterfaceAudience.Private + public static + InetAddress getByName(String hostname) throws UnknownHostException { + return hostResolver.getByName(hostname); + } + + interface HostResolver { + InetAddress getByName(String host) throws UnknownHostException; + } + + /** + * Uses standard java host resolution + */ + static class StandardHostResolver implements HostResolver { + public InetAddress getByName(String host) throws UnknownHostException { + return InetAddress.getByName(host); + } + } + + /** + * This an alternate resolver with important properties that the standard + * java resolver lacks: + * 1) The hostname is fully qualified. This avoids security issues if not + * all hosts in the cluster do not share the same search domains. It + * also prevents other hosts from performing unnecessary dns searches. + * In contrast, InetAddress simply returns the host as given. + * 2) The InetAddress is instantiated with an exact host and IP to prevent + * further unnecessary lookups. InetAddress may perform an unnecessary + * reverse lookup for an IP. + * 3) A call to getHostName() will always return the qualified hostname, or + * more importantly, the IP if instantiated with an IP. This avoids + * unnecessary dns timeouts if the host is not resolvable. + * 4) Point 3 also ensures that if the host is re-resolved, ex. during a + * connection re-attempt, that a reverse lookup to host and forward + * lookup to IP is not performed since the reverse/forward mappings may + * not always return the same IP. If the client initiated a connection + * with an IP, then that IP is all that should ever be contacted. + * + * NOTE: this resolver is only used if: + * hadoop.security.token.service.use_ip=false + */ + protected static class QualifiedHostResolver implements HostResolver { + @SuppressWarnings("unchecked") + private List<String> searchDomains = + ResolverConfiguration.open().searchlist(); + + /** + * Create an InetAddress with a fully qualified hostname of the given + * hostname. InetAddress does not qualify an incomplete hostname that + * is resolved via the domain search list. + * {@link InetAddress#getCanonicalHostName()} will fully qualify the + * hostname, but it always return the A record whereas the given hostname + * may be a CNAME. + * + * @param host a hostname or ip address + * @return InetAddress with the fully qualified hostname or ip + * @throws UnknownHostException if host does not exist + */ + public InetAddress getByName(String host) throws UnknownHostException { + InetAddress addr = null; + + if (IPAddressUtil.isIPv4LiteralAddress(host)) { + // use ipv4 address as-is + byte[] ip = IPAddressUtil.textToNumericFormatV4(host); + addr = InetAddress.getByAddress(host, ip); + } else if (IPAddressUtil.isIPv6LiteralAddress(host)) { + // use ipv6 address as-is + byte[] ip = IPAddressUtil.textToNumericFormatV6(host); + addr = InetAddress.getByAddress(host, ip); + } else if (host.endsWith(".")) { + // a rooted host ends with a dot, ex. "host." + // rooted hosts never use the search path, so only try an exact lookup + addr = getByExactName(host); + } else if (host.contains(".")) { + // the host contains a dot (domain), ex. "host.domain" + // try an exact host lookup, then fallback to search list + addr = getByExactName(host); + if (addr == null) { + addr = getByNameWithSearch(host); + } + } else { + // it's a simple host with no dots, ex. "host" + // try the search list, then fallback to exact host + InetAddress loopback = InetAddress.getByName(null); + if (host.equalsIgnoreCase(loopback.getHostName())) { + addr = InetAddress.getByAddress(host, loopback.getAddress()); + } else { + addr = getByNameWithSearch(host); + if (addr == null) { + addr = getByExactName(host); + } + } + } + // unresolvable! + if (addr == null) { + throw new UnknownHostException(host); + } + return addr; + } + + InetAddress getByExactName(String host) { + InetAddress addr = null; + // InetAddress will use the search list unless the host is rooted + // with a trailing dot. The trailing dot will disable any use of the + // search path in a lower level resolver. See RFC 1535. + String fqHost = host; + if (!fqHost.endsWith(".")) fqHost += "."; + try { + addr = getInetAddressByName(fqHost); + // can't leave the hostname as rooted or other parts of the system + // malfunction, ex. kerberos principals are lacking proper host + // equivalence for rooted/non-rooted hostnames + addr = InetAddress.getByAddress(host, addr.getAddress()); + } catch (UnknownHostException e) { + // ignore, caller will throw if necessary + } + return addr; + } + + InetAddress getByNameWithSearch(String host) { + InetAddress addr = null; + if (host.endsWith(".")) { // already qualified? + addr = getByExactName(host); + } else { + for (String domain : searchDomains) { + String dot = !domain.startsWith(".") ? "." : ""; + addr = getByExactName(host + dot + domain); + if (addr != null) break; + } + } + return addr; + } + + // implemented as a separate method to facilitate unit testing + InetAddress getInetAddressByName(String host) throws UnknownHostException { + return InetAddress.getByName(host); + } + + void setSearchDomains(String ... domains) { + searchDomains = Arrays.asList(domains); + } + } } Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=1235107&r1=1235106&r2=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileSystem.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileSystem.java Tue Jan 24 01:24:53 2012 @@ -60,7 +60,7 @@ import org.apache.hadoop.mapred.Reporter import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.lib.LongSumReducer; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.net.NetUtilsTestResolver; +import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/TestNetUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/TestNetUtils.java?rev=1235107&r1=1235106&r2=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/TestNetUtils.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/TestNetUtils.java Tue Jan 24 01:24:53 2012 @@ -33,6 +33,7 @@ import java.net.UnknownHostException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.NetUtilsTestResolver; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; Copied: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/NetUtilsTestResolver.java (from r1235092, hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/NetUtilsTestResolver.java) URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/NetUtilsTestResolver.java?p2=hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/NetUtilsTestResolver.java&p1=hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/NetUtilsTestResolver.java&r1=1235092&r2=1235107&rev=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/net/NetUtilsTestResolver.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/NetUtilsTestResolver.java Tue Jan 24 01:24:53 2012 @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.net; +package org.apache.hadoop.security; import java.net.InetAddress; import java.net.UnknownHostException; @@ -25,7 +25,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import org.apache.hadoop.net.NetUtils.QualifiedHostResolver; +import org.apache.hadoop.security.SecurityUtil.QualifiedHostResolver; /** * provides a dummy dns search resolver with a configurable search path @@ -41,7 +41,7 @@ public class NetUtilsTestResolver extend resolver.addResolvedHost("host.a.b.", "1.1.1.1"); resolver.addResolvedHost("b-host.b.", "2.2.2.2"); resolver.addResolvedHost("simple.", "3.3.3.3"); - NetUtils.setHostResolver(resolver); + SecurityUtil.hostResolver = resolver; return resolver; } @@ -56,7 +56,8 @@ public class NetUtilsTestResolver extend resolvedHosts.put(host, addr); } - InetAddress getInetAddressByName(String host) throws UnknownHostException { + @Override + public InetAddress getInetAddressByName(String host) throws UnknownHostException { hostSearches.add(host); if (!resolvedHosts.containsKey(host)) { throw new UnknownHostException(host); @@ -64,11 +65,21 @@ public class NetUtilsTestResolver extend return resolvedHosts.get(host); } - String[] getHostSearches() { + @Override + public InetAddress getByExactName(String host) { + return super.getByExactName(host); + } + + @Override + public InetAddress getByNameWithSearch(String host) { + return super.getByNameWithSearch(host); + } + + public String[] getHostSearches() { return hostSearches.toArray(new String[0]); } - void reset() { + public void reset() { hostSearches.clear(); } -} +} \ No newline at end of file Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java?rev=1235107&r1=1235106&r2=1235107&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/security/TestSecurityUtil.java Tue Jan 24 01:24:53 2012 @@ -200,7 +200,7 @@ public class TestSecurityUtil { assertTrue(!addr.isUnresolved()); // don't know what the standard resolver will return for hostname. // should be host for host; host or ip for ip is ambiguous - if (!SecurityUtil.getTokenServiceUseIp()) { + if (!SecurityUtil.useIpForTokenService) { assertEquals(host, addr.getHostName()); assertEquals(host, addr.getAddress().getHostName()); }