hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sandeep Jangra <sandeepjan...@gmail.com>
Subject Re: Trouble with Word Count example
Date Thu, 29 Nov 2012 18:36:12 GMT
Also, I did set the HADOOP_CLASSPATH variable to point to the word_cnt.jar
only.


On Thu, Nov 29, 2012 at 10:54 AM, Sandeep Jangra <sandeepjangra@gmail.com>wrote:

> Thanks for the quick response Mahesh.
>
> I am using the following command:
>
> sudo -u hdfs hadoop jar word_cnt.jar WordCount2  /tmp/root/input
> /tmp/root/output15  -libjars=word_cnt.jar
> (The input directory exists on the hdfs)
>
> This is how I compiled and packaged it:
>
> javac -classpath
> /usr/lib/hadoop-0.20-mapreduce/hadoop-core.jar:/usr/lib/hadoop/*  -d
> word_cnt WordCount2.java
> jar -cvf word_cnt.jar -C word_cnt/ .
>
>
>
>
> On Thu, Nov 29, 2012 at 10:46 AM, Mahesh Balija <
> balijamahesh.mca@gmail.com> wrote:
>
>> Hi Sandeep,
>>
>>            For me everything seems to be alright.
>>            Can you tell us how are you running this job?
>>
>> Best,
>> Mahesh.B.
>> Calsoft Labs.
>>
>> On Thu, Nov 29, 2012 at 9:01 PM, Sandeep Jangra <sandeepjangra@gmail.com>wrote:
>>
>>> Hello everyone,
>>>
>>>   Like most others I am also running into some problems while running my
>>> word count example.
>>>   I tried the various suggestion available on internet, but I guess it;s
>>> time to go on email :)
>>>
>>>   Here is the error that I am getting:
>>>   12/11/29 10:20:59 WARN mapred.JobClient: Use GenericOptionsParser for
>>> parsing the arguments. Applications should implement Tool for the same.
>>> 12/11/29 10:20:59 WARN mapred.JobClient: No job jar file set.  User
>>> classes may not be found. See JobConf(Class) or JobConf#setJar(String).
>>> 12/11/29 10:20:59 INFO input.FileInputFormat: Total input paths to
>>> process : 1
>>> 12/11/29 10:20:59 INFO util.NativeCodeLoader: Loaded the native-hadoop
>>> library
>>> 12/11/29 10:20:59 WARN snappy.LoadSnappy: Snappy native library is
>>> available
>>> 12/11/29 10:20:59 INFO snappy.LoadSnappy: Snappy native library loaded
>>> 12/11/29 10:21:00 INFO mapred.JobClient: Running job:
>>> job_201210310210_0040
>>> 12/11/29 10:21:01 INFO mapred.JobClient:  map 0% reduce 0%
>>> 12/11/29 10:21:07 INFO mapred.JobClient: Task Id :
>>> attempt_201210310210_0040_m_000000_0, Status : FAILED
>>> java.lang.RuntimeException: *java.lang.ClassNotFoundException: Class
>>> WordCount2$Map *not found
>>> at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1439)
>>> at
>>> org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:191)
>>> at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:605)
>>> at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
>>> at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
>>> at java.security.AccessController.doPrivileged(Native Method)
>>> at javax.security.auth.Subject.doAs(Subject.java:416)
>>> at
>>> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1232)
>>> at org.apache.hadoop.mapred.Child.main(Child.java:264)
>>> Caused by: java.lang.ClassNotFoundException: Class WordCount2$Map not
>>> found
>>> at
>>> org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1350)
>>> at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1437)
>>> ... 8 more
>>>
>>> And here is the source code:
>>>
>>>
>>> import org.apache.hadoop.conf.Configuration;
>>> import org.apache.hadoop.conf.Configured;
>>> import org.apache.hadoop.fs.Path;
>>> import org.apache.hadoop.io.IntWritable;
>>> import org.apache.hadoop.io.LongWritable;
>>> import org.apache.hadoop.io.Text;
>>> import org.apache.hadoop.mapreduce.Job;
>>> import org.apache.hadoop.mapreduce.Mapper;
>>> import org.apache.hadoop.mapreduce.Reducer;
>>> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
>>> import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
>>> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>>> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>>> import org.apache.hadoop.util.Tool;
>>> import org.apache.hadoop.util.ToolRunner;
>>>
>>> import java.io.IOException;
>>> import java.util.StringTokenizer;
>>>
>>> public class WordCount2 extends Configured implements Tool {
>>>
>>>
>>>     public static class Map extends Mapper<LongWritable, Text, Text,
>>> IntWritable> {
>>>         private final static IntWritable one = new IntWritable(1);
>>>         private Text word = new Text();
>>>
>>>         @Override
>>>         protected void map(LongWritable key, Text value, Context
>>> context) throws IOException, InterruptedException {
>>>             String line = value.toString();
>>>             StringTokenizer tokenizer = new StringTokenizer(line);
>>>             while (tokenizer.hasMoreTokens()) {
>>>                 word.set(tokenizer.nextToken());
>>>                 context.write(word, one);
>>>             }
>>>         }
>>>
>>>     }
>>>
>>>
>>>     public static class Reduce extends Reducer<Text, IntWritable, Text,
>>> IntWritable> {
>>>
>>>         @Override
>>>         protected void reduce(Text key, Iterable<IntWritable> values,
>>> Context context) throws IOException, InterruptedException {
>>>
>>>             int sum = 0;
>>>
>>>             for(IntWritable value : values) {
>>>                 sum += value.get();
>>>             }
>>> //                    while (values.hasNext()) {
>>> //                          sum += values.next().get();
>>> //                        }
>>>             context.write(key, new IntWritable(sum));
>>>         }
>>>
>>>     }
>>>
>>>     @Override
>>>     public int run(String[] args) throws Exception {
>>>         Configuration conf = getConf();
>>>         for (java.util.Map.Entry<String, String> entry: conf) {
>>>             System.out.printf("%s=%s\n", entry.getKey(),
>>> entry.getValue());
>>>         }
>>>
>>>         System.out.println("arg[0]= "+args[0] + " args[1]= "+ args[1]);
>>>
>>>         Job job = new Job(conf, WordCount2.class.getSimpleName());
>>>         job.setJobName("wordcount2");
>>>         job.setJarByClass(WordCount2.class);
>>>
>>>         job.setMapOutputKeyClass(Text.class);
>>>         job.setMapOutputValueClass(IntWritable.class);
>>>
>>>         job.setOutputKeyClass(Text.class);
>>>         job.setOutputValueClass(IntWritable.class);
>>>
>>>         job.setMapperClass(Map.class);
>>>         job.setCombinerClass(Reduce.class);
>>>         job.setReducerClass(Reduce.class);
>>>
>>>         job.setInputFormatClass(TextInputFormat.class);
>>>         job.setOutputFormatClass(TextOutputFormat.class);
>>>
>>>         FileInputFormat.setInputPaths(job, new Path(args[0]));
>>>         FileOutputFormat.setOutputPath(job, new Path(args[1]));
>>>
>>>         System.exit(job.waitForCompletion(true) ? 0 : 1);
>>>
>>>         return 0;
>>>     }
>>>
>>>
>>>
>>>     public static void main(String[] args) throws Exception {
>>>         int exitCode = ToolRunner.run(new WordCount2(), args);
>>>         System.exit(exitCode);
>>>     }
>>> }
>>>
>>>
>>>
>>
>

Mime
View raw message