Mailing-List: contact user-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: user@hadoop.apache.org
Received-SPF: pass (athena.apache.org: domain of sandeepjangra@gmail.com
 designates 209.85.216.176 as permitted sender)
MIME-Version: 1.0
Date: Thu, 29 Nov 2012 10:31:00 -0500
Message-ID: 
 <CAGuT+87GE=jArh-a8Xh9eGnq2V=O-dFeLu2jEuJm0chOYeRNUQ@mail.gmail.com>
Subject: Trouble with Word Count example
From: Sandeep Jangra <sandeepjangra@gmail.com>
To: user@hadoop.apache.org
Content-Type: multipart/alternative; boundary=047d7b6dcbd66046fc04cfa3f85b

--047d7b6dcbd66046fc04cfa3f85b
Content-Type: text/plain; charset=ISO-8859-1

Hello everyone,

  Like most others I am also running into some problems while running my
word count example.
  I tried the various suggestion available on internet, but I guess it;s
time to go on email :)

  Here is the error that I am getting:
  12/11/29 10:20:59 WARN mapred.JobClient: Use GenericOptionsParser for
parsing the arguments. Applications should implement Tool for the same.
12/11/29 10:20:59 WARN mapred.JobClient: No job jar file set.  User classes
may not be found. See JobConf(Class) or JobConf#setJar(String).
12/11/29 10:20:59 INFO input.FileInputFormat: Total input paths to process
: 1
12/11/29 10:20:59 INFO util.NativeCodeLoader: Loaded the native-hadoop
library
12/11/29 10:20:59 WARN snappy.LoadSnappy: Snappy native library is available
12/11/29 10:20:59 INFO snappy.LoadSnappy: Snappy native library loaded
12/11/29 10:21:00 INFO mapred.JobClient: Running job: job_201210310210_0040
12/11/29 10:21:01 INFO mapred.JobClient:  map 0% reduce 0%
12/11/29 10:21:07 INFO mapred.JobClient: Task Id :
attempt_201210310210_0040_m_000000_0, Status : FAILED
java.lang.RuntimeException: *java.lang.ClassNotFoundException: Class
WordCount2$Map *not found
at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1439)
 at
org.apache.hadoop.mapreduce.task.JobContextImpl.getMapperClass(JobContextImpl.java:191)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:605)
 at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
at org.apache.hadoop.mapred.Child$4.run(Child.java:270)
 at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:416)
 at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1232)
at org.apache.hadoop.mapred.Child.main(Child.java:264)
Caused by: java.lang.ClassNotFoundException: Class WordCount2$Map not found
at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1350)
 at org.apache.hadoop.conf.Configuration.getClass(Configuration.java:1437)
... 8 more

And here is the source code:


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import java.io.IOException;
import java.util.StringTokenizer;

public class WordCount2 extends Configured implements Tool {


    public static class Map extends Mapper<LongWritable, Text, Text,
IntWritable> {
        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        @Override
        protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
            String line = value.toString();
            StringTokenizer tokenizer = new StringTokenizer(line);
            while (tokenizer.hasMoreTokens()) {
                word.set(tokenizer.nextToken());
                context.write(word, one);
            }
        }

    }


    public static class Reduce extends Reducer<Text, IntWritable, Text,
IntWritable> {

        @Override
        protected void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {

            int sum = 0;

            for(IntWritable value : values) {
                sum += value.get();
            }
//                    while (values.hasNext()) {
//                          sum += values.next().get();
//                        }
            context.write(key, new IntWritable(sum));
        }

    }

    @Override
    public int run(String[] args) throws Exception {
        Configuration conf = getConf();
        for (java.util.Map.Entry<String, String> entry: conf) {
            System.out.printf("%s=%s\n", entry.getKey(), entry.getValue());
        }

        System.out.println("arg[0]= "+args[0] + " args[1]= "+ args[1]);

        Job job = new Job(conf, WordCount2.class.getSimpleName());
        job.setJobName("wordcount2");
        job.setJarByClass(WordCount2.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setMapperClass(Map.class);
        job.setCombinerClass(Reduce.class);
        job.setReducerClass(Reduce.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        System.exit(job.waitForCompletion(true) ? 0 : 1);

        return 0;
    }


    public static void main(String[] args) throws Exception {
        int exitCode = ToolRunner.run(new WordCount2(), args);
        System.exit(exitCode);
    }
}

--047d7b6dcbd66046fc04cfa3f85b
Content-Type: text/html; charset=ISO-8859-1
Content-Transfer-Encoding: quoted-printable

Hello everyone,<div><br></div><div>=A0 Like most others I am also running i=
nto some problems while running my word count example.</div><div>=A0 I trie=
d the various suggestion available on internet, but I guess it;s time to go=
 on email :)</div>

<div>=A0=A0</div><div>=A0 Here is the error that I am getting:</div><div>=
=A0=A012/11/29 10:20:59 WARN mapred.JobClient: Use GenericOptionsParser for=
 parsing the arguments. Applications should implement Tool for the same.</d=
iv><div>

12/11/29 10:20:59 WARN mapred.JobClient: No job jar file set. =A0User class=
es may not be found. See JobConf(Class) or JobConf#setJar(String).</div><di=
v>12/11/29 10:20:59 INFO input.FileInputFormat: Total input paths to proces=
s : 1</div>

<div>12/11/29 10:20:59 INFO util.NativeCodeLoader: Loaded the native-hadoop=
 library</div><div>12/11/29 10:20:59 WARN snappy.LoadSnappy: Snappy native =
library is available</div><div>12/11/29 10:20:59 INFO snappy.LoadSnappy: Sn=
appy native library loaded</div>

<div>12/11/29 10:21:00 INFO mapred.JobClient: Running job: job_201210310210=
_0040</div><div>12/11/29 10:21:01 INFO mapred.JobClient: =A0map 0% reduce 0=
%</div><div>12/11/29 10:21:07 INFO mapred.JobClient: Task Id : attempt_2012=
10310210_0040_m_000000_0, Status : FAILED</div>

<div>java.lang.RuntimeException: <b>java.lang.ClassNotFoundException: Class=
 WordCount2$Map </b>not found</div><div><span style=3D"white-space:pre-wrap=
">	</span>at org.apache.hadoop.conf.Configuration.getClass(Configuration.ja=
va:1439)</div>

<div><span style=3D"white-space:pre-wrap">	</span>at org.apache.hadoop.mapr=
educe.task.JobContextImpl.getMapperClass(JobContextImpl.java:191)</div><div=
><span style=3D"white-space:pre-wrap">	</span>at org.apache.hadoop.mapred.M=
apTask.runNewMapper(MapTask.java:605)</div>

<div><span style=3D"white-space:pre-wrap">	</span>at org.apache.hadoop.mapr=
ed.MapTask.run(MapTask.java:325)</div><div><span style=3D"white-space:pre-w=
rap">	</span>at org.apache.hadoop.mapred.Child$4.run(Child.java:270)</div>

<div><span style=3D"white-space:pre-wrap">	</span>at java.security.AccessCo=
ntroller.doPrivileged(Native Method)</div><div><span style=3D"white-space:p=
re-wrap">	</span>at javax.security.auth.Subject.doAs(Subject.java:416)</div=
>

<div><span style=3D"white-space:pre-wrap">	</span>at org.apache.hadoop.secu=
rity.UserGroupInformation.doAs(UserGroupInformation.java:1232)</div><div><s=
pan style=3D"white-space:pre-wrap">	</span>at org.apache.hadoop.mapred.Chil=
d.main(Child.java:264)</div>

<div>Caused by: java.lang.ClassNotFoundException: Class WordCount2$Map not =
found</div><div><span style=3D"white-space:pre-wrap">	</span>at org.apache.=
hadoop.conf.Configuration.getClassByName(Configuration.java:1350)</div>
<div><span style=3D"white-space:pre-wrap">	</span>at org.apache.hadoop.conf=
.Configuration.getClass(Configuration.java:1437)</div><div><span style=3D"w=
hite-space:pre-wrap">	</span>... 8 more</div>
<div><br></div><div>And here is the source code:</div><div><br></div><div><=
div><br></div><div>import org.apache.hadoop.conf.Configuration;</div><div>i=
mport org.apache.hadoop.conf.Configured;</div><div>import org.apache.hadoop=
.fs.Path;</div>

<div>import org.apache.hadoop.io.IntWritable;</div><div>import org.apache.h=
adoop.io.LongWritable;</div><div>import org.apache.hadoop.io.Text;</div><di=
v>import org.apache.hadoop.mapreduce.Job;</div><div>import org.apache.hadoo=
p.mapreduce.Mapper;</div>

<div>import org.apache.hadoop.mapreduce.Reducer;</div><div>import org.apach=
e.hadoop.mapreduce.lib.input.FileInputFormat;</div><div>import org.apache.h=
adoop.mapreduce.lib.input.TextInputFormat;</div><div>import org.apache.hado=
op.mapreduce.lib.output.FileOutputFormat;</div>

<div>import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;</div><=
div>import org.apache.hadoop.util.Tool;</div><div>import org.apache.hadoop.=
util.ToolRunner;</div><div><br></div><div>import java.io.IOException;</div>

<div>import java.util.StringTokenizer;</div><div><br></div><div>public clas=
s WordCount2 extends Configured implements Tool {</div><div><br></div><div>=
<br></div><div>=A0 =A0 public static class Map extends Mapper&lt;LongWritab=
le, Text, Text, IntWritable&gt; {</div>

<div>=A0 =A0 =A0 =A0 private final static IntWritable one =3D new IntWritab=
le(1);</div><div>=A0 =A0 =A0 =A0 private Text word =3D new Text();</div><di=
v><br></div><div>=A0 =A0 =A0 =A0 @Override</div><div>=A0 =A0 =A0 =A0 protec=
ted void map(LongWritable key, Text value, Context context) throws IOExcept=
ion, InterruptedException {</div>

<div>=A0 =A0 =A0 =A0 =A0 =A0 String line =3D value.toString();</div><div>=
=A0 =A0 =A0 =A0 =A0 =A0 StringTokenizer tokenizer =3D new StringTokenizer(l=
ine);</div><div>=A0 =A0 =A0 =A0 =A0 =A0 while (tokenizer.hasMoreTokens()) {=
</div><div>=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 word.set(tokenizer.nextToken());=
</div>

<div>=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 context.write(word, one);</div><div>=
=A0 =A0 =A0 =A0 =A0 =A0 }</div><div>=A0 =A0 =A0 =A0 }</div><div><br></div><=
div>=A0 =A0 }</div><div><br></div><div><br></div><div>=A0 =A0 public static=
 class Reduce extends Reducer&lt;Text, IntWritable, Text, IntWritable&gt; {=
</div>

<div><br></div><div>=A0 =A0 =A0 =A0 @Override</div><div>=A0 =A0 =A0 =A0 pro=
tected void reduce(Text key, Iterable&lt;IntWritable&gt; values, Context co=
ntext) throws IOException, InterruptedException {</div><div><br></div><div>=
=A0 =A0 =A0 =A0 =A0 =A0 int sum =3D 0;</div>

<div><br></div><div>=A0 =A0 =A0 =A0 =A0 =A0 for(IntWritable value : values)=
 {</div><div>=A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 sum +=3D value.get();</div><di=
v>=A0 =A0 =A0 =A0 =A0 =A0 }</div><div>// =A0 =A0 =A0 =A0 =A0 =A0<span style=
=3D"white-space:pre-wrap">	</span> =A0 =A0 =A0 =A0while (values.hasNext()) =
{</div>

<div>// =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0<span style=3D"white-space:pre-wrap"=
>	</span> =A0 =A0 =A0 =A0 =A0sum +=3D values.next().get();</div><div>// =A0=
 =A0 =A0 =A0 =A0 =A0 =A0 =A0<span style=3D"white-space:pre-wrap">	</span> =
=A0 =A0 =A0 =A0}</div>
<div>=A0 =A0 =A0 =A0 =A0 =A0 context.write(key, new IntWritable(sum));</div=
><div>=A0 =A0 =A0 =A0 }</div><div><br></div><div>=A0 =A0 }</div><div><br></=
div><div>=A0 =A0 @Override</div><div>=A0 =A0 public int run(String[] args) =
throws Exception {</div><div>

=A0 =A0 =A0 =A0 Configuration conf =3D getConf();</div><div>=A0 =A0 =A0 =A0=
 for (java.util.Map.Entry&lt;String, String&gt; entry: conf) {</div><div>=
=A0 =A0 =A0 =A0 =A0 =A0 System.out.printf(&quot;%s=3D%s\n&quot;, entry.getK=
ey(), entry.getValue());</div>

<div>=A0 =A0 =A0 =A0 }</div><div><br></div><div>=A0 =A0 =A0 =A0 System.out.=
println(&quot;arg[0]=3D &quot;+args[0] + &quot; args[1]=3D &quot;+ args[1])=
;</div><div><br></div><div>=A0 =A0 =A0 =A0 Job job =3D new Job(conf, WordCo=
unt2.class.getSimpleName());</div>

<div>=A0 =A0 =A0 =A0 job.setJobName(&quot;wordcount2&quot;);</div><div>=A0 =
=A0 =A0 =A0 job.setJarByClass(WordCount2.class);</div><div><br></div><div>=
=A0 =A0 =A0 =A0 job.setMapOutputKeyClass(Text.class);</div><div>=A0 =A0 =A0=
 =A0 job.setMapOutputValueClass(IntWritable.class);</div>

<div><br></div><div>=A0 =A0 =A0 =A0 job.setOutputKeyClass(Text.class);</div=
><div>=A0 =A0 =A0 =A0 job.setOutputValueClass(IntWritable.class);</div><div=
><br></div><div>=A0 =A0 =A0 =A0 job.setMapperClass(Map.class);</div><div>=
=A0 =A0 =A0 =A0 job.setCombinerClass(Reduce.class);</div>

<div>=A0 =A0 =A0 =A0 job.setReducerClass(Reduce.class);</div><div><br></div=
><div>=A0 =A0 =A0 =A0 job.setInputFormatClass(TextInputFormat.class);</div>=
<div>=A0 =A0 =A0 =A0 job.setOutputFormatClass(TextOutputFormat.class);</div=
><div><br></div><div>

=A0 =A0 =A0 =A0 FileInputFormat.setInputPaths(job, new Path(args[0]));</div=
><div>=A0 =A0 =A0 =A0 FileOutputFormat.setOutputPath(job, new Path(args[1])=
);</div><div><br></div><div>=A0 =A0 =A0 =A0 System.exit(job.waitForCompleti=
on(true) ? 0 : 1);</div>

<div><br></div><div>=A0 =A0 =A0 =A0 return 0;</div><div>=A0 =A0 }</div><div=
><br></div><div><br></div><div><br></div><div>=A0 =A0 public static void ma=
in(String[] args) throws Exception {</div><div>=A0 =A0 =A0 =A0 int exitCode=
 =3D ToolRunner.run(new WordCount2(), args);</div>

<div>=A0 =A0 =A0 =A0 System.exit(exitCode);</div><div>=A0 =A0 }</div><div>}=
</div></div><div><br></div><div><br></div>

--047d7b6dcbd66046fc04cfa3f85b--