hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sandeep Jangra <sandeepjan...@gmail.com>
Subject Trouble with Word Count example
Date Thu, 29 Nov 2012 15:31:00 GMT
Hello everyone,

  Like most others I am also running into some problems while running my
word count example.
  I tried the various suggestion available on internet, but I guess it;s
time to go on email :)

  Here is the error that I am getting:
  12/11/29 10:20:59 WARN mapred.JobClient: Use GenericOptionsParser for
parsing the arguments. Applications should implement Tool for the same.
12/11/29 10:20:59 WARN mapred.JobClient: No job jar file set.  User classes
may not be found. See JobConf(Class) or JobConf#setJar(String).
12/11/29 10:20:59 INFO input.FileInputFormat: Total input paths to process
: 1
12/11/29 10:20:59 INFO util.NativeCodeLoader: Loaded the native-hadoop
library
12/11/29 10:20:59 WARN snappy.LoadSnappy: Snappy native library is available
12/11/29 10:20:59 INFO snappy.LoadSnappy: Snappy native library loaded
12/11/29 10:21:00 INFO mapred.JobClient: Running job: job_201210310210_0040
12/11/29 10:21:01 INFO mapred.JobClient:  map 0% reduce 0%
12/11/29 10:21:07 INFO mapred.JobClient: Task Id :
attempt_201210310210_0040_m_000000_0, Status : FAILED
java.lang.RuntimeException: *java.lang.ClassNotFoundException: Class
WordCount2$Map *not found
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1439)
 at
org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:191)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:605)
 at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
 at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:416)
 at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1232)
at org.apache.hadoop.mapred.Child.main(Child.java:264)
Caused by: java.lang.ClassNotFoundException: Class WordCount2$Map not found
at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1350)
 at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1437)
... 8 more

And here is the source code:


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;
import java.util.StringTokenizer;

public class WordCount2 extends Configured implements Tool {


    public static class Map extends Mapper<LongWritable, Text, Text,
IntWritable> {
        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        @Override
        protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
            String line = value.toString();
            StringTokenizer tokenizer = new StringTokenizer(line);
            while (tokenizer.hasMoreTokens()) {
                word.set(tokenizer.nextToken());
                context.write(word, one);
            }
        }

    }


    public static class Reduce extends Reducer<Text, IntWritable, Text,
IntWritable> {

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {

            int sum = 0;

            for(IntWritable value : values) {
                sum += value.get();
            }
//                    while (values.hasNext()) {
//                          sum += values.next().get();
//                        }
            context.write(key, new IntWritable(sum));
        }

    }

    @Override
    public int run(String[] args) throws Exception {
        Configuration conf = getConf();
        for (java.util.Map.Entry<String, String> entry: conf) {
            System.out.printf("%s=%s\n", entry.getKey(), entry.getValue());
        }

        System.out.println("arg[0]= "+args[0] + " args[1]= "+ args[1]);

        Job job = new Job(conf, WordCount2.class.getSimpleName());
        job.setJobName("wordcount2");
        job.setJarByClass(WordCount2.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setMapperClass(Map.class);
        job.setCombinerClass(Reduce.class);
        job.setReducerClass(Reduce.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

        return 0;
    }



    public static void main(String[] args) throws Exception {
        int exitCode = ToolRunner.run(new WordCount2(), args);
        System.exit(exitCode);
    }
}

Mime
View raw message