Author: cnauroth Date: Mon Dec 9 22:52:02 2013 New Revision: 1549699 URL: http://svn.apache.org/r1549699 Log: Merge trunk to HDFS-4685.
Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1549699&r1=1549698&r2=1549699&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt Mon Dec 9 22:52:02 2013 @@ -487,6 +487,9 @@ Release 2.3.0 - UNRELEASED OPTIMIZATIONS + HADOOP-10142. Avoid groups lookup for unprivileged users such as "dr.who" + (vinay via cmccabe) + BUG FIXES HADOOP-10028. Malformed ssl-server.xml.example. (Haohui Mai via jing9) Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1548386-1549698 Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1548386-1549698 Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1548386-1549698 Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1549699&r1=1549698&r2=1549699&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon Dec 9 22:52:02 2013 @@ -204,6 +204,14 @@ public class CommonConfigurationKeys ext public static final String DEFAULT_HADOOP_HTTP_STATIC_USER = "dr.who"; + /** + * User->groups static mapping to override the groups lookup + */ + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES = + "hadoop.user.group.static.mapping.overrides"; + public static final String HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT = + "dr.who=;"; + /** Enable/Disable aliases serving from jetty */ public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES = "hadoop.jetty.logs.serve.aliases"; Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java?rev=1549699&r1=1549698&r2=1549699&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java Mon Dec 9 22:52:02 2013 @@ -18,15 +18,20 @@ package org.apache.hadoop.security; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.commons.logging.Log; @@ -49,6 +54,8 @@ public class Groups { private final Map<String, CachedGroups> userToGroupsMap = new ConcurrentHashMap<String, CachedGroups>(); + private final Map<String, List<String>> staticUserToGroupsMap = + new HashMap<String, List<String>>(); private final long cacheTimeout; private final long warningDeltaMs; @@ -66,12 +73,43 @@ public class Groups { warningDeltaMs = conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS, CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT); - + parseStaticMapping(conf); + if(LOG.isDebugEnabled()) LOG.debug("Group mapping impl=" + impl.getClass().getName() + "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" + warningDeltaMs); } + + /* + * Parse the hadoop.user.group.static.mapping.overrides configuration to + * staticUserToGroupsMap + */ + private void parseStaticMapping(Configuration conf) { + String staticMapping = conf.get( + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT); + Collection<String> mappings = StringUtils.getStringCollection( + staticMapping, ";"); + for (String users : mappings) { + Collection<String> userToGroups = StringUtils.getStringCollection(users, + "="); + if (userToGroups.size() < 1 || userToGroups.size() > 2) { + throw new HadoopIllegalArgumentException("Configuration " + + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES + + " is invalid"); + } + String[] userToGroupsArray = userToGroups.toArray(new String[userToGroups + .size()]); + String user = userToGroupsArray[0]; + List<String> groups = Collections.emptyList(); + if (userToGroupsArray.length == 2) { + groups = (List<String>) StringUtils + .getStringCollection(userToGroupsArray[1]); + } + staticUserToGroupsMap.put(user, groups); + } + } /** * Get the group memberships of a given user. @@ -80,6 +118,11 @@ public class Groups { * @throws IOException */ public List<String> getGroups(String user) throws IOException { + // No need to lookup for groups of static users + List<String> staticMapping = staticUserToGroupsMap.get(user); + if (staticMapping != null) { + return staticMapping; + } // Return cached value if available CachedGroups groups = userToGroupsMap.get(user); long startMs = Time.monotonicNow(); Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1549699&r1=1549698&r2=1549699&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Mon Dec 9 22:52:02 2013 @@ -325,10 +325,24 @@ public class StringUtils { * @return an <code>ArrayList</code> of string values */ public static Collection<String> getStringCollection(String str){ + String delim = ","; + return getStringCollection(str, delim); + } + + /** + * Returns a collection of strings. + * + * @param str + * String to parse + * @param delim + * delimiter to separate the values + * @return Collection of parsed elements. + */ + public static Collection<String> getStringCollection(String str, String delim) { List<String> values = new ArrayList<String>(); if (str == null) return values; - StringTokenizer tokenizer = new StringTokenizer (str,","); + StringTokenizer tokenizer = new StringTokenizer(str, delim); values = new ArrayList<String>(); while (tokenizer.hasMoreTokens()) { values.add(tokenizer.nextToken()); Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1549699&r1=1549698&r2=1549699&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Mon Dec 9 22:52:02 2013 @@ -1261,4 +1261,18 @@ Specify the port number used by Hadoop mount daemon. </description> </property> + +<property> + <name>hadoop.user.group.static.mapping.overrides</name> + <value>dr.who=;</value> + <description> + Static mapping of user to groups. This will override the groups if + available in the system for the specified user. In otherwords, groups + look-up will not happen for these users, instead groups mapped in this + configuration will be used. + Mapping should be in this format. + user1=group1,group2;user2=;user3=group2; + Default, "dr.who=;" will consider "dr.who" as user without groups. + </description> +</property> </configuration> Propchange: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1548386-1549698 Modified: hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java?rev=1549699&r1=1549698&r2=1549699&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java Mon Dec 9 22:52:02 2013 @@ -19,14 +19,17 @@ package org.apache.hadoop.security; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; +import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import org.apache.commons.logging.Log; @@ -40,10 +43,12 @@ import org.apache.hadoop.security.ShellB public class TestGroupsCaching { public static final Log LOG = LogFactory.getLog(TestGroupsCaching.class); - private static Configuration conf = new Configuration(); private static String[] myGroups = {"grp1", "grp2"}; + private Configuration conf; - static { + @Before + public void setup() { + conf = new Configuration(); conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, FakeGroupMapping.class, ShellBasedUnixGroupsMapping.class); @@ -88,7 +93,7 @@ public class TestGroupsCaching { } @Test - public void TestGroupsCaching() throws Exception { + public void testGroupsCaching() throws Exception { Groups groups = new Groups(conf); groups.cacheGroupsAdd(Arrays.asList(myGroups)); groups.refresh(); @@ -117,4 +122,45 @@ public class TestGroupsCaching { FakeGroupMapping.clearBlackList(); assertTrue(groups.getGroups("user1").size() == 2); } + + public static class FakeunPrivilegedGroupMapping extends FakeGroupMapping { + private static boolean invoked = false; + @Override + public List<String> getGroups(String user) throws IOException { + invoked = true; + return super.getGroups(user); + } + } + + /* + * Group lookup should not happen for static users + */ + @Test + public void testGroupLookupForStaticUsers() throws Exception { + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + FakeunPrivilegedGroupMapping.class, ShellBasedUnixGroupsMapping.class); + conf.set(CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, "me=;user1=group1;user2=group1,group2"); + Groups groups = new Groups(conf); + List<String> userGroups = groups.getGroups("me"); + assertTrue("non-empty groups for static user", userGroups.isEmpty()); + assertFalse("group lookup done for static user", + FakeunPrivilegedGroupMapping.invoked); + + List<String> expected = new ArrayList<String>(); + expected.add("group1"); + + FakeunPrivilegedGroupMapping.invoked = false; + userGroups = groups.getGroups("user1"); + assertTrue("groups not correct", expected.equals(userGroups)); + assertFalse("group lookup done for unprivileged user", + FakeunPrivilegedGroupMapping.invoked); + + expected.add("group2"); + FakeunPrivilegedGroupMapping.invoked = false; + userGroups = groups.getGroups("user2"); + assertTrue("groups not correct", expected.equals(userGroups)); + assertFalse("group lookup done for unprivileged user", + FakeunPrivilegedGroupMapping.invoked); + + } }