hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Vitaliy Semochkin <vitaliy...@gmail.com>
Subject why my Reduce Class does not work?
Date Sun, 04 Jul 2010 17:32:01 GMT
Hi,

I rewritten WordCount sample to use new Hadoop API

however my reduce task doesn't launch.

the result file always looks like
some_word 1
some_word 1
another_word 1
another_word 1

...

Here is the code:

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

public static class WordCountMapper extends Mapper<LongWritable, Text, Text,
IntWritable> {

@Override
protected void map(LongWritable key, Text value, Context context) throws
IOException, InterruptedException {
StringTokenizer st = new StringTokenizer(value.toString());
while (st.hasMoreTokens()) {
context.write(new Text(st.nextToken()), new IntWritable(1));
}
}
}

public static class WordCountReduce extends Reducer<Text, IntWritable, Text,
IntWritable> {

@SuppressWarnings("unchecked")
public void reduce(Text key, Iterable<IntWritable> values, Reducer.Context
context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable value : values) {
sum += value.get();
}
context.write(key, new IntWritable(sum));
}
}

public static void main(String[] args) throws IOException,
InterruptedException, ClassNotFoundException {
Job job = new Job();
job.setJobName("WordCounter");
job.setJarByClass(WordCount.class);
job.setMapperClass(WordCountMapper.class);
job.setReducerClass(WordCountReduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true)? 0 :1);
}
}

Looks like WordCountReduce was never launched but I don't see any Warnings
or Errors in log  file.

Any help is highly appreciated.

Thanks in Advance,
Vitaliy S

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message