---------- Forwarded message ---------- From: air <cnwe...@gmail.com> Date: 2011/7/28 Subject: HBase Mapreduce cannot find Map class To: CDH Users <cdh-u...@cloudera.org>
import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.lib.NullOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; public class LoadToHBase extends Configured implements Tool{ public static class XMap<K, V> extends MapReduceBase implements Mapper<LongWritable, Text, K, V>{ private JobConf conf; @Override public void configure(JobConf conf){ this.conf = conf; try{ this.table = new HTable(new HBaseConfiguration(conf), "observations"); }catch(IOException e){ throw new RuntimeException("Failed HTable construction", e); } } @Override public void close() throws IOException{ super.close(); table.close(); } private HTable table; public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException{ String[] valuelist = value.toString().split("\t"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Date addtime = null; // 用户注册时间 Date ds = null; Long delta_days = null; String uid = valuelist[0]; try { addtime = sdf.parse(valuelist[1]); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } String ds_str = conf.get("load.hbase.ds", null); if (ds_str != null){ try { ds = sdf.parse(ds_str); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } }else{ ds_str = "2011-07-28"; } if (addtime != null && ds != null){ delta_days = (ds.getTime() - addtime.getTime()) / (24 * 60 * 60 * 1000); } if (delta_days != null){ byte[] rowKey = uid.getBytes(); Put p = new Put(rowKey); p.add("content".getBytes(), "attr1".getBytes(), delta_days.toString().getBytes()); table.put(p); } } } /** * @param args * @throws Exception */ public static void main(String[] args) throws Exception { // TODO Auto-generated method stub int exitCode = ToolRunner.run(new HBaseConfiguration(), new LoadToHBase(), args); System.exit(exitCode); } @Override public int run(String[] args) throws Exception { // TODO Auto-generated method stub JobConf conf = new JobConf(getClass()); TableMapReduceUtil.addDependencyJars(conf); FileInputFormat.addInputPath(conf, new Path(args[0])); conf.setJobName("LoadToHBase"); conf.setJarByClass(getClass()); conf.setMapperClass(XMap.class); conf.setNumReduceTasks(0); conf.setOutputFormat(NullOutputFormat.class); JobClient.runJob(conf); return 0; } } execute it using hbase LoadToHBase /user/hive/warehouse/datamining.db/xxx/ and it says: ...... 11/07/28 17:20:29 INFO mapred.JobClient: Task Id : attempt_201107261532_2625_m_000004_1, Status : FAILED java.lang.RuntimeException: Error in configuring object at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) at org.apache.hadoop.mapred.Child$4.run(Child.java:270) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127) at org.apache.hadoop.mapred.Child.main(Child.java:264) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) ... 9 more Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028) at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968) at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) ... 14 more Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020) ... 16 more Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap at java.net.URLClassLoader$1.run(URLClassLoader.java:202) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:190) at java.lang.ClassLoader.loadClass(ClassLoader.java:306) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) at java.lang.ClassLoader.loadClass(ClassLoader.java:247) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:247) at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994) ... 17 more 11/07/28 17:20:29 INFO mapred.JobClient: Task Id : attempt_201107261532_2625_m_000006_1, Status : FAILED java.lang.RuntimeException: Error in configuring object at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) at org.apache.hadoop.mapred.Child$4.run(Child.java:270) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:396) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127) at org.apache.hadoop.mapred.Child.main(Child.java:264) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) ... 9 more Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028) at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968) at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) ... 14 more Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: LoadToHBase$XMap at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020) ... 16 more Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap at java.net.URLClassLoader$1.run(URLClassLoader.java:202) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:190) at java.lang.ClassLoader.loadClass(ClassLoader.java:306) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) at java.lang.ClassLoader.loadClass(ClassLoader.java:247) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:247) at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943) at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994) 11/07/28 17:20:29 INFO mapred.JobClient: Job complete: job_201107261532_2625 11/07/28 17:20:29 INFO mapred.JobClient: Counters: 8 11/07/28 17:20:29 INFO mapred.JobClient: Job Counters 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=1504081 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all reduces waiting after reserving slots (ms)=0 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all maps waiting after reserving slots (ms)=0 11/07/28 17:20:29 INFO mapred.JobClient: Rack-local map tasks=187 11/07/28 17:20:29 INFO mapred.JobClient: Launched map tasks=350 11/07/28 17:20:29 INFO mapred.JobClient: Data-local map tasks=163 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=0 11/07/28 17:20:29 INFO mapred.JobClient: Failed map tasks=1 11/07/28 17:20:29 INFO mapred.JobClient: Job Failed: NA Exception in thread "main" java.io.IOException: Job failed! at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1246) at LoadToHBase.run(LoadToHBase.java:106) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65) at LoadToHBase.main(LoadToHBase.java:91) why it cannot find XMap, it's a inner class, it should be found ... I am really confused with this situation... thank you for you help -- Knowledge Mangement . -- Knowledge Mangement .