hadoop-hdfs-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Harsh J <ha...@cloudera.com>
Subject Re: Reducer not getting called
Date Thu, 13 Jun 2013 05:30:40 GMT
You're not using the recommended @Override annotations, and are
hitting a classic programming mistake. Your issue is same as this
earlier discussion: http://search-hadoop.com/m/gqA3rAaVQ7 (and the
ones before it).

On Thu, Jun 13, 2013 at 9:52 AM, Omkar Joshi
<Omkar.Joshi@lntinfotech.com> wrote:
> Hi,
>
>
>
> I have a SequenceFile which contains several jpeg images with (image name,
> image bytes) as key-value pairs. My objective is to count the no. of images
> by grouping them by the source, something like this :
>
>
>
> Nikon Coolpix  100
>
> Sony Cybershot 251
>
> N82 100
>
>
>
>
>
> The MR code is :
>
>
>
> package com.hadoop.basics;
>
>
>
> import java.io.BufferedInputStream;
>
> import java.io.ByteArrayInputStream;
>
> import java.io.IOException;
>
> import java.util.Iterator;
>
>
>
> import org.apache.hadoop.conf.Configuration;
>
> import org.apache.hadoop.conf.Configured;
>
> import org.apache.hadoop.fs.Path;
>
> import org.apache.hadoop.io.BytesWritable;
>
> import org.apache.hadoop.io.IntWritable;
>
> import org.apache.hadoop.io.Text;
>
> import org.apache.hadoop.mapreduce.Job;
>
> import org.apache.hadoop.mapreduce.Mapper;
>
> import org.apache.hadoop.mapreduce.Reducer;
>
> import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
>
> import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
>
> import org.apache.hadoop.util.Tool;
>
> import org.apache.hadoop.util.ToolRunner;
>
>
>
> import com.drew.imaging.ImageMetadataReader;
>
> import com.drew.imaging.ImageProcessingException;
>
> import com.drew.metadata.Directory;
>
> import com.drew.metadata.Metadata;
>
> import com.drew.metadata.exif.ExifIFD0Directory;
>
>
>
> public class ImageSummary extends Configured implements Tool {
>
>
>
>             public static class ImageSourceMapper extends
>
>                                     Mapper<Text, BytesWritable, Text,
> IntWritable> {
>
>
>
>                         private static int tagId = 272;
>
>                         private static final IntWritable one = new
> IntWritable(1);
>
>
>
>                         public void map(Text imageName, BytesWritable
> imageBytes,
>
>                                                 Context context) throws
> IOException, InterruptedException {
>
>                                     // TODO Auto-generated method stub
>
>
>
>                                     System.out.println("In the map method,
> image is "
>
>                                                             +
> imageName.toString());
>
>
>
>                                     byte[] imageInBytes =
> imageBytes.getBytes();
>
>                                     ByteArrayInputStream bais = new
> ByteArrayInputStream(imageInBytes);
>
>                                     BufferedInputStream bis = new
> BufferedInputStream(bais);
>
>
>
>                                     Metadata imageMD = null;
>
>
>
>                                     try {
>
>                                                 imageMD =
> ImageMetadataReader.readMetadata(bis, true);
>
>                                     } catch (ImageProcessingException e) {
>
>                                                 // TODO Auto-generated catch
> block
>
>                                                 System.out.println("Got an
> ImageProcessingException !");
>
>                                                 e.printStackTrace();
>
>                                     }
>
>
>
>                                     Directory exifIFD0Directory = imageMD
>
>
> .getDirectory(ExifIFD0Directory.class);
>
>
>
>                                     String imageSource =
> exifIFD0Directory.getString(tagId);
>
>
>
>                                     System.out.println(imageName.toString()
> + " is taken using "
>
>                                                             + imageSource);
>
>
>
>                                     context.write(new Text(imageSource),
> one);
>
>
>
>                                     System.out.println("Returning from the
> map method");
>
>                         }
>
>             }
>
>
>
>             public static class ImageSourceReducer extends
>
>                                     Reducer<Text, IntWritable, Text,
> IntWritable> {
>
>
>
>                         public void reduce(Text imageSource,
> Iterator<IntWritable> counts,
>
>                                                 Context context) throws
> IOException, InterruptedException {
>
>                                     // TODO Auto-generated method stub
>
>
>
>                                     System.out.println("In the reduce
> method");
>
>
>
>                                     int finalCount = 0;
>
>
>
>                                     while (counts.hasNext()) {
>
>                                                 finalCount +=
> counts.next().get();
>
>                                     }
>
>
>
>                                     context.write(imageSource, new
> IntWritable(finalCount));
>
>
>
>                                     System.out.println("Returning from the
> reduce method");
>
>                         }
>
>
>
>             }
>
>
>
>             public static void main(String[] args) throws Exception {
>
>                         ToolRunner.run(new ImageSummary(), args);
>
>             }
>
>
>
>             @Override
>
>             public int run(String[] args) throws Exception {
>
>                         // TODO Auto-generated method stub
>
>
>
>                         System.out.println("In ImageSummary.run(...)");
>
>
>
>                         Configuration configuration = getConf();
>
>
>
>                         Job job = new Job(configuration, "Image_Source");
>
>                         job.setJarByClass(getClass());
>
>
>
>
> job.setInputFormatClass(SequenceFileInputFormat.class);
>
>                         job.setOutputFormatClass(TextOutputFormat.class);
>
>
>
>                         job.setMapperClass(ImageSourceMapper.class);
>
>                         job.setCombinerClass(ImageSourceReducer.class);
>
>                         job.setReducerClass(ImageSourceReducer.class);
>
>
>
>                         job.setMapOutputKeyClass(Text.class);
>
>                         job.setMapOutputValueClass(IntWritable.class);
>
>
>
>                         job.setOutputKeyClass(Text.class);
>
>                         job.setOutputValueClass(IntWritable.class);
>
>
>
>                         SequenceFileInputFormat.addInputPath(job, new
> Path(args[0]));
>
>                         TextOutputFormat.setOutputPath(job, new
> Path(args[1]));
>
>
>
>                         System.out.println("Submitting job");
>
>
>
>                         job.waitForCompletion(true);
>
>
>
>                         int jobStatus = job.isSuccessful() ? 0 : -1;
>
>
>
>                         System.out.println("Returning jobStatus = " +
> jobStatus);
>
>
>
>                         return jobStatus;
>
>             }
>
> }
>
>
>
>
>
> The command :
>
>
>
> hadoop jar /home/hduser/dumphere/codes/hadoop/imageops.jar
> com.hadoop.basics.ImageSummary "/scratchpad/imageOps/WholeImageSeqFile"
> "/scratchpad/imageOps/cnt"
>
>
>
>
>
> The part-file (/scratchpad/imageOps/cnt/part-r-00000)
>
> COOLPIX L120             1
>
> COOLPIX L120             1
>
> K750i 1
>
>
>
> The mapper stdout logs :
>
>
>
> stdout logs
>
> In the map method, image is It's a long road....JPG
>
> It's a long road....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is Every man is a mountainside....JPG
>
> Every man is a mountainside....JPG is taken using COOLPIX L120
>
> Returning from the map method
>
> In the map method, image is mystic.JPG
>
> mystic.JPG is taken using K750i
>
> Returning from the map method
>
> But nothing is reflected in stdout logs of the reducer.
>
> What have I missed?
>
>
>
> Regards,
>
> Omkar Joshi
>
>
>
>
>
>
> ________________________________
> The contents of this e-mail and any attachment(s) may contain confidential
> or privileged information for the intended recipient(s). Unintended
> recipients are prohibited from taking action on the basis of information in
> this e-mail and using or disseminating the information, and must notify the
> sender and delete it from their system. L&T Infotech will not accept
> responsibility or liability for the accuracy or completeness of, or the
> presence of any virus or disabling code in this e-mail"



-- 
Harsh J

Mime
View raw message