Author: todd Date: Wed Feb 15 18:10:21 2012 New Revision: 1244620 URL: http://svn.apache.org/viewvc?rev=1244620&view=rev Log: HADOOP-6502. Improve the performance of Configuration.getClassByName when the class is not found by caching negative results. Contributed by Sharad Agarwal and Todd Lipcon.
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1244620&r1=1244619&r2=1244620&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Feb 15 18:10:21 2012 @@ -169,6 +169,10 @@ Release 0.23.2 - UNRELEASED HADOOP-8071. Avoid an extra packet in client code when nagling is disabled. (todd) + HADOOP-6502. Improve the performance of Configuration.getClassByName when + the class is not found by caching negative results. + (sharad, todd via todd) + BUG FIXES HADOOP-8042 When copying a file out of HDFS, modifying it, and uploading Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1244620&r1=1244619&r2=1244620&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Wed Feb 15 18:10:21 2012 @@ -1146,6 +1146,22 @@ public class Configuration implements It * @throws ClassNotFoundException if the class is not found. */ public Class<?> getClassByName(String name) throws ClassNotFoundException { + Class<?> ret = getClassByNameOrNull(name); + if (ret == null) { + throw new ClassNotFoundException("Class " + name + " not found"); + } + return ret; + } + + /** + * Load a class by name, returning null rather than throwing an exception + * if it couldn't be loaded. This is to avoid the overhead of creating + * an exception. + * + * @param name the class name + * @return the class object, or null if it could not be found. + */ + public Class<?> getClassByNameOrNull(String name) { Map<String, Class<?>> map; synchronized (CACHE_CLASSES) { @@ -1157,12 +1173,20 @@ public class Configuration implements It } } - Class<?> clazz = map.get(name); - if (clazz == null) { - clazz = Class.forName(name, true, classLoader); - if (clazz != null) { - // two putters can race here, but they'll put the same class - map.put(name, clazz); + Class<?> clazz = null; + if (!map.containsKey(name)) { + try { + clazz = Class.forName(name, true, classLoader); + } catch (ClassNotFoundException e) { + map.put(name, null); //cache negative that class is not found + return null; + } + // two putters can race here, but they'll put the same class + map.put(name, clazz); + } else { // check already performed on this class name + clazz = map.get(name); + if (clazz == null) { // found the negative + return null; } } Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java?rev=1244620&r1=1244619&r2=1244620&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java Wed Feb 15 18:10:21 2012 @@ -86,17 +86,22 @@ public class ReflectionUtils { //invoke configure on theObject try { Class<?> jobConfClass = - conf.getClassByName("org.apache.hadoop.mapred.JobConf"); + conf.getClassByNameOrNull("org.apache.hadoop.mapred.JobConf"); + if (jobConfClass == null) { + return; + } + Class<?> jobConfigurableClass = - conf.getClassByName("org.apache.hadoop.mapred.JobConfigurable"); - if (jobConfClass.isAssignableFrom(conf.getClass()) && + conf.getClassByNameOrNull("org.apache.hadoop.mapred.JobConfigurable"); + if (jobConfigurableClass == null) { + return; + } + if (jobConfClass.isAssignableFrom(conf.getClass()) && jobConfigurableClass.isAssignableFrom(theObject.getClass())) { Method configureMethod = jobConfigurableClass.getMethod("configure", jobConfClass); configureMethod.invoke(theObject, conf); } - } catch (ClassNotFoundException e) { - //JobConf/JobConfigurable not in classpath. no need to configure } catch (Exception e) { throw new RuntimeException("Error in configuring object", e); }