See http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/package-summary.html#package_description for some help. St.Ack
On Thu, Jul 28, 2011 at 4:04 AM, air <cnwe...@gmail.com> wrote: > ---------- Forwarded message ---------- > From: air <cnwe...@gmail.com> > Date: 2011/7/28 > Subject: HBase Mapreduce cannot find Map class > To: CDH Users <cdh-u...@cloudera.org> > > > import java.io.IOException; > import java.text.ParseException; > import java.text.SimpleDateFormat; > import java.util.Date; > > import org.apache.hadoop.conf.Configured; > import org.apache.hadoop.fs.Path; > import org.apache.hadoop.hbase.HBaseConfiguration; > import org.apache.hadoop.hbase.client.HTable; > import org.apache.hadoop.hbase.client.Put; > import org.apache.hadoop.hbase.mapred.TableMapReduceUtil; > import org.apache.hadoop.io.LongWritable; > import org.apache.hadoop.io.Text; > import org.apache.hadoop.mapred.JobClient; > import org.apache.hadoop.mapred.JobConf; > import org.apache.hadoop.mapred.MapReduceBase; > import org.apache.hadoop.mapred.Mapper; > import org.apache.hadoop.mapred.OutputCollector; > import org.apache.hadoop.mapred.Reporter; > import org.apache.hadoop.mapred.FileInputFormat; > import org.apache.hadoop.mapred.lib.NullOutputFormat; > import org.apache.hadoop.util.Tool; > import org.apache.hadoop.util.ToolRunner; > > > public class LoadToHBase extends Configured implements Tool{ > public static class XMap<K, V> extends MapReduceBase implements > Mapper<LongWritable, Text, K, V>{ > private JobConf conf; > > @Override > public void configure(JobConf conf){ > this.conf = conf; > try{ > this.table = new HTable(new HBaseConfiguration(conf), > "observations"); > }catch(IOException e){ > throw new RuntimeException("Failed HTable construction", e); > } > } > > @Override > public void close() throws IOException{ > super.close(); > table.close(); > } > > private HTable table; > public void map(LongWritable key, Text value, OutputCollector > output, Reporter reporter) throws IOException{ > String[] valuelist = value.toString().split("\t"); > SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd > HH:mm:ss"); > Date addtime = null; // 用户注册时间 > Date ds = null; > Long delta_days = null; > String uid = valuelist[0]; > try { > addtime = sdf.parse(valuelist[1]); > } catch (ParseException e) { > // TODO Auto-generated catch block > e.printStackTrace(); > } > > String ds_str = conf.get("load.hbase.ds", null); > if (ds_str != null){ > try { > ds = sdf.parse(ds_str); > } catch (ParseException e) { > // TODO Auto-generated catch block > e.printStackTrace(); > } > }else{ > ds_str = "2011-07-28"; > } > > if (addtime != null && ds != null){ > delta_days = (ds.getTime() - addtime.getTime()) / (24 * 60 * > 60 * 1000); > } > > if (delta_days != null){ > byte[] rowKey = uid.getBytes(); > Put p = new Put(rowKey); > p.add("content".getBytes(), "attr1".getBytes(), > delta_days.toString().getBytes()); > table.put(p); > } > } > } > /** > * @param args > * @throws Exception > */ > public static void main(String[] args) throws Exception { > // TODO Auto-generated method stub > int exitCode = ToolRunner.run(new HBaseConfiguration(), new > LoadToHBase(), args); > System.exit(exitCode); > } > > @Override > public int run(String[] args) throws Exception { > // TODO Auto-generated method stub > JobConf conf = new JobConf(getClass()); > TableMapReduceUtil.addDependencyJars(conf); > FileInputFormat.addInputPath(conf, new Path(args[0])); > conf.setJobName("LoadToHBase"); > conf.setJarByClass(getClass()); > conf.setMapperClass(XMap.class); > conf.setNumReduceTasks(0); > conf.setOutputFormat(NullOutputFormat.class); > JobClient.runJob(conf); > return 0; > } > > } > > execute it using hbase LoadToHBase /user/hive/warehouse/datamining.db/xxx/ > and it says: > > ...... > 11/07/28 17:20:29 INFO mapred.JobClient: Task Id : > attempt_201107261532_2625_m_000004_1, Status : FAILED > java.lang.RuntimeException: Error in configuring object > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387) > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) > at org.apache.hadoop.mapred.Child$4.run(Child.java:270) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:396) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127) > at org.apache.hadoop.mapred.Child.main(Child.java:264) > Caused by: java.lang.reflect.InvocationTargetException > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > at java.lang.reflect.Method.invoke(Method.java:597) > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) > ... 9 more > Caused by: java.lang.RuntimeException: java.lang.RuntimeException: > java.lang.ClassNotFoundException: LoadToHBase$XMap > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028) > at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968) > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ... 14 more > Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: > LoadToHBase$XMap > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996) > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020) > ... 16 more > Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap > at java.net.URLClassLoader$1.run(URLClassLoader.java:202) > at java.security.AccessController.doPrivileged(Native Method) > at java.net.URLClassLoader.findClass(URLClassLoader.java:190) > at java.lang.ClassLoader.loadClass(ClassLoader.java:306) > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) > at java.lang.ClassLoader.loadClass(ClassLoader.java:247) > at java.lang.Class.forName0(Native Method) > at java.lang.Class.forName(Class.java:247) > at > org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943) > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994) > ... 17 more > > 11/07/28 17:20:29 INFO mapred.JobClient: Task Id : > attempt_201107261532_2625_m_000006_1, Status : FAILED > java.lang.RuntimeException: Error in configuring object > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:93) > at > org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:64) > at > org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:117) > at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:387) > at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325) > at org.apache.hadoop.mapred.Child$4.run(Child.java:270) > at java.security.AccessController.doPrivileged(Native Method) > at javax.security.auth.Subject.doAs(Subject.java:396) > at > org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1127) > at org.apache.hadoop.mapred.Child.main(Child.java:264) > Caused by: java.lang.reflect.InvocationTargetException > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) > at java.lang.reflect.Method.invoke(Method.java:597) > at > org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:88) > ... 9 more > Caused by: java.lang.RuntimeException: java.lang.RuntimeException: > java.lang.ClassNotFoundException: LoadToHBase$XMap > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1028) > at org.apache.hadoop.mapred.JobConf.getMapperClass(JobConf.java:968) > at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:34) > ... 14 more > Caused by: java.lang.RuntimeException: java.lang.ClassNotFoundException: > LoadToHBase$XMap > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:996) > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1020) > ... 16 more > Caused by: java.lang.ClassNotFoundException: LoadToHBase$XMap > at java.net.URLClassLoader$1.run(URLClassLoader.java:202) > at java.security.AccessController.doPrivileged(Native Method) > at java.net.URLClassLoader.findClass(URLClassLoader.java:190) > at java.lang.ClassLoader.loadClass(ClassLoader.java:306) > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301) > at java.lang.ClassLoader.loadClass(ClassLoader.java:247) > at java.lang.Class.forName0(Native Method) > at java.lang.Class.forName(Class.java:247) > at > org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:943) > at > org.apache.hadoop.conf.Configuration.getClass(Configuration.java:994) > > 11/07/28 17:20:29 INFO mapred.JobClient: Job complete: job_201107261532_2625 > 11/07/28 17:20:29 INFO mapred.JobClient: Counters: 8 > 11/07/28 17:20:29 INFO mapred.JobClient: Job Counters > 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=1504081 > 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all reduces > waiting after reserving slots (ms)=0 > 11/07/28 17:20:29 INFO mapred.JobClient: Total time spent by all maps > waiting after reserving slots (ms)=0 > 11/07/28 17:20:29 INFO mapred.JobClient: Rack-local map tasks=187 > 11/07/28 17:20:29 INFO mapred.JobClient: Launched map tasks=350 > 11/07/28 17:20:29 INFO mapred.JobClient: Data-local map tasks=163 > 11/07/28 17:20:29 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=0 > 11/07/28 17:20:29 INFO mapred.JobClient: Failed map tasks=1 > 11/07/28 17:20:29 INFO mapred.JobClient: Job Failed: NA > Exception in thread "main" java.io.IOException: Job failed! > at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1246) > at LoadToHBase.run(LoadToHBase.java:106) > at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:65) > at LoadToHBase.main(LoadToHBase.java:91) > > > why it cannot find XMap, it's a inner class, it should be found ... I am > really confused with this situation... thank you for you help > -- > Knowledge Mangement . > > > > > -- > Knowledge Mangement . >