Hadoop Example in java
Hi All, I am looking for example in java for hadoop. I have done lots of search but I have only found word count. Are there any other exapmple for the same. -- View this message in context: http://old.nabble.com/Hadoop-Example-in-java-tp33341353p33341353.html Sent from the Hadoop core-user mailing list archive at Nabble.com.
Re: Hadoop Example in java
Hi Vikas,
You could also try out various examples like finding the maximum temperature
from a given dataset
006701199091950051507004...999N9+1+999...
004301199091950051512004...999N9+00221+999...
004301199091950051518004...999N9-00111+999...
004301265091949032412004...051N9+0+999...
004301265091949032418004...051N9+00781+999...
//Mapper for maximum temperature example
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class MaxTemperatureMapper
extends Mapper {
private static final int MISSING = ;
@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
}
}
String line = value.toString();
String year = line.substring(15, 19);
int airTemperature;
if (line.charAt(87) == '+') { // parseInt doesn't like leading plus signs
airTemperature = Integer.parseInt(line.substring(88, 92));
} else {
airTemperature = Integer.parseInt(line.substring(87, 92));
}
String quality = line.substring(92, 93);
if (airTemperature != MISSING && quality.matches("[01459]")) {
context.write(new Text(year), new IntWritable(airTemperature));
}
}}
//Reducer for maximum temperature example
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class MaxTemperatureReducer
extends Reducer {
@Override
public void reduce(Text key, Iterable values,
Context context)
throws IOException, InterruptedException {
}
}
int maxValue = Integer.MIN_VALUE;
for (IntWritable value : values) {
maxValue = Math.max(maxValue, value.get());
}
context.write(key, new IntWritable(maxValue));
}
}
//Application to find the maximum temperature in the weather dataset
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class MaxTemperature {
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: MaxTemperature ");
System.exit(-1);
}
Job job = new Job();
job.setJarByClass(MaxTemperature.class);
job.setJobName("Max temperature");
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(MaxTemperatureMapper.class);
job.setReducerClass(MaxTemperatureReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
}
}
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
--
View this message in context:
http://old.nabble.com/Hadoop-Example-in-java-tp33341353p34208568.html
Sent from the Hadoop core-user mailing list archive at Nabble.com.
Re: Hadoop Example in java
HI vikas,
You can download example programes from facebook group link below:
http://www.facebook.com/groups/416125741763625/
It contain some ppt as well.
Regards,
Saravanan Nagarajan
On Wed, Jul 25, 2012 at 10:17 AM, minumichael wrote:
>
> Hi Vikas,
>
> You could also try out various examples like finding the maximum
> temperature
> from a given dataset
>
> 006701199091950051507004...999N9+1+999...
> 004301199091950051512004...999N9+00221+999...
> 004301199091950051518004...999N9-00111+999...
> 004301265091949032412004...051N9+0+999...
> 004301265091949032418004...051N9+00781+999...
>
> //Mapper for maximum temperature example
>
> import java.io.IOException;
> import org.apache.hadoop.io.LongWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapreduce.Mapper;
>
> public class MaxTemperatureMapper
> extends Mapper {
> private static final int MISSING = ;
> @Override
> public void map(LongWritable key, Text value, Context context)
> throws IOException, InterruptedException {
> }
> }
> String line = value.toString();
> String year = line.substring(15, 19);
> int airTemperature;
> if (line.charAt(87) == '+') { // parseInt doesn't like leading plus signs
> airTemperature = Integer.parseInt(line.substring(88, 92));
> } else {
> airTemperature = Integer.parseInt(line.substring(87, 92));
> }
> String quality = line.substring(92, 93);
> if (airTemperature != MISSING && quality.matches("[01459]")) {
> context.write(new Text(year), new IntWritable(airTemperature));
> }
> }}
> //Reducer for maximum temperature example
> import java.io.IOException;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapreduce.Reducer;
> public class MaxTemperatureReducer
> extends Reducer {
> @Override
> public void reduce(Text key, Iterable values,
> Context context)
> throws IOException, InterruptedException {
> }
> }
> int maxValue = Integer.MIN_VALUE;
> for (IntWritable value : values) {
> maxValue = Math.max(maxValue, value.get());
> }
> context.write(key, new IntWritable(maxValue));
> }
> }
> //Application to find the maximum temperature in the weather dataset
> import org.apache.hadoop.fs.Path;
> import org.apache.hadoop.io.IntWritable;
> import org.apache.hadoop.io.Text;
> import org.apache.hadoop.mapreduce.Job;
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
> public class MaxTemperature {
> public static void main(String[] args) throws Exception {
> if (args.length != 2) {
> System.err.println("Usage: MaxTemperature ");
> System.exit(-1);
> }
> Job job = new Job();
> job.setJarByClass(MaxTemperature.class);
> job.setJobName("Max temperature");
> FileInputFormat.addInputPath(job, new Path(args[0]));
> FileOutputFormat.setOutputPath(job, new Path(args[1]));
> job.setMapperClass(MaxTemperatureMapper.class);
> job.setReducerClass(MaxTemperatureReducer.class);
> job.setOutputKeyClass(Text.class);
> job.setOutputValueClass(IntWritable.class);
> }
> }
> System.exit(job.waitForCompletion(true) ? 0 : 1);
> }
> }
>
> --
> View this message in context:
> http://old.nabble.com/Hadoop-Example-in-java-tp33341353p34208568.html
> Sent from the Hadoop core-user mailing list archive at Nabble.com.
>
>
Re: Hadoop Example in java
http://lintool.github.com/MapReduceAlgorithms/index.html
The original book for a lot of really cool map-reduce algorithms.
After you "get" what these classes do, get Hive and Pig. They both
have an 'explain plan' command that shows you the chain of map-reduce
jobs needed for your high-level code. Really helpful.
On Tue, Jul 24, 2012 at 10:13 PM, Saravanan Nagarajan
wrote:
> HI vikas,
>
> You can download example programes from facebook group link below:
> http://www.facebook.com/groups/416125741763625/
>
>
> It contain some ppt as well.
>
> Regards,
> Saravanan Nagarajan
>
> On Wed, Jul 25, 2012 at 10:17 AM, minumichael > wrote:
>
>>
>> Hi Vikas,
>>
>> You could also try out various examples like finding the maximum
>> temperature
>> from a given dataset
>>
>> 006701199091950051507004...999N9+1+999...
>> 004301199091950051512004...999N9+00221+999...
>> 004301199091950051518004...999N9-00111+999...
>> 004301265091949032412004...051N9+0+999...
>> 004301265091949032418004...051N9+00781+999...
>>
>> //Mapper for maximum temperature example
>>
>> import java.io.IOException;
>> import org.apache.hadoop.io.LongWritable;
>> import org.apache.hadoop.io.Text;
>> import org.apache.hadoop.mapreduce.Mapper;
>>
>> public class MaxTemperatureMapper
>> extends Mapper {
>> private static final int MISSING = ;
>> @Override
>> public void map(LongWritable key, Text value, Context context)
>> throws IOException, InterruptedException {
>> }
>> }
>> String line = value.toString();
>> String year = line.substring(15, 19);
>> int airTemperature;
>> if (line.charAt(87) == '+') { // parseInt doesn't like leading plus signs
>> airTemperature = Integer.parseInt(line.substring(88, 92));
>> } else {
>> airTemperature = Integer.parseInt(line.substring(87, 92));
>> }
>> String quality = line.substring(92, 93);
>> if (airTemperature != MISSING && quality.matches("[01459]")) {
>> context.write(new Text(year), new IntWritable(airTemperature));
>> }
>> }}
>> //Reducer for maximum temperature example
>> import java.io.IOException;
>> import org.apache.hadoop.io.IntWritable;
>> import org.apache.hadoop.io.Text;
>> import org.apache.hadoop.mapreduce.Reducer;
>> public class MaxTemperatureReducer
>> extends Reducer {
>> @Override
>> public void reduce(Text key, Iterable values,
>> Context context)
>> throws IOException, InterruptedException {
>> }
>> }
>> int maxValue = Integer.MIN_VALUE;
>> for (IntWritable value : values) {
>> maxValue = Math.max(maxValue, value.get());
>> }
>> context.write(key, new IntWritable(maxValue));
>> }
>> }
>> //Application to find the maximum temperature in the weather dataset
>> import org.apache.hadoop.fs.Path;
>> import org.apache.hadoop.io.IntWritable;
>> import org.apache.hadoop.io.Text;
>> import org.apache.hadoop.mapreduce.Job;
>> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
>> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>> public class MaxTemperature {
>> public static void main(String[] args) throws Exception {
>> if (args.length != 2) {
>> System.err.println("Usage: MaxTemperature ");
>> System.exit(-1);
>> }
>> Job job = new Job();
>> job.setJarByClass(MaxTemperature.class);
>> job.setJobName("Max temperature");
>> FileInputFormat.addInputPath(job, new Path(args[0]));
>> FileOutputFormat.setOutputPath(job, new Path(args[1]));
>> job.setMapperClass(MaxTemperatureMapper.class);
>> job.setReducerClass(MaxTemperatureReducer.class);
>> job.setOutputKeyClass(Text.class);
>> job.setOutputValueClass(IntWritable.class);
>> }
>> }
>> System.exit(job.waitForCompletion(true) ? 0 : 1);
>> }
>> }
>>
>> --
>> View this message in context:
>> http://old.nabble.com/Hadoop-Example-in-java-tp33341353p34208568.html
>> Sent from the Hadoop core-user mailing list archive at Nabble.com.
>>
>>
--
Lance Norskog
goks...@gmail.com
Re: Hadoop Example in java
For more framework-provided examples, also take a look at your downloaded distributions' src/examples directory. I also suggest getting 'Hadoop: The Definitive Guide" by Tom White (O'Reilly) to get started with, it carries examples and all other information useful for using/deploying/developing with Apache Hadoop. On Fri, Feb 17, 2012 at 2:30 PM, vikas jain wrote: > > Hi All, > > I am looking for example in java for hadoop. I have done lots of search but > I have only found word count. Are there any other exapmple for the same. > -- > View this message in context: > http://old.nabble.com/Hadoop-Example-in-java-tp33341353p33341353.html > Sent from the Hadoop core-user mailing list archive at Nabble.com. > -- Harsh J Customer Ops. Engineer Cloudera | http://tiny.cloudera.com/about
Re: Hadoop Example in java
On Fri, Feb 17, 2012 at 1:00 AM, vikas jain wrote: > > Hi All, > > I am looking for example in java for hadoop. I have done lots of search but > I have only found word count. Are there any other exapmple for the same. If you want to find them on the web, you can look in subversion: http://svn.apache.org/repos/asf/hadoop/common/branches/branch-1/src/examples/org/apache/hadoop/examples/ -- Owen
