hadoop-mapreduce-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ranjini (JIRA)" <j...@apache.org>
Subject [jira] [Created] (MAPREDUCE-5666) org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):cannot find symbol
Date Wed, 04 Dec 2013 05:49:37 GMT
ranjini created MAPREDUCE-5666:
----------------------------------

             Summary: org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):cannot
find symbol
                 Key: MAPREDUCE-5666
                 URL: https://issues.apache.org/jira/browse/MAPREDUCE-5666
             Project: Hadoop Map/Reduce
          Issue Type: Bug
            Reporter: ranjini


hi 

I have written the below code , and facing the issue. i am using hadoop 0.20 vesion and java
1.6 the issue is 

org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):232:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
          if (globStat.isDirectory()) {
                      ^
org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java(org/apache/hadoop/mapreduce/lib/output:FileOutputCommitter.java):208:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
    } else if(fs.getFileStatus(taskOutput).isDirectory()) {
                                          ^
org/apache/hadoop/mapred/JobConf.java(org/apache/hadoop/mapred:JobConf.java):433: cannot find
symbol
symbol  : method getPattern(java.lang.String,java.util.regex.Pattern)
location: class org.apache.hadoop.mapred.JobConf
    return getPattern(JobContext.JAR_UNPACK_PATTERN, UNPACK_JAR_PATTERN_DEFAULT);
           ^
org/apache/hadoop/mapred/JobConf.java(org/apache/hadoop/mapred:JobConf.java):450: cannot find
symbol
symbol  : method getTrimmedStrings(java.lang.String)
location: class org.apache.hadoop.mapred.JobConf
    return getTrimmedStrings(MRConfig.LOCAL_DIR);
           ^
org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):165:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
      if (stat.isDirectory()) {
              ^
org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):215:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
          if (globStat.isDirectory()) {
                      ^
org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):218:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
              if (recursive && stat.isDirectory()) {
                                   ^
org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):258:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
      if (file.isDirectory()) {
              ^
org/apache/hadoop/mapred/FileOutputCommitter.java(org/apache/hadoop/mapred:FileOutputCommitter.java):166:
cannot find symbol
symbol  : method isDirectory()
location: class org.apache.hadoop.fs.FileStatus
    } else if(fs.getFileStatus(taskOutput).isDirectory()) {
                                          ^
org/apache/hadoop/mapred/LineRecordReader.java(org/apache/hadoop/mapred:LineRecordReader.java):100:
incompatible types
found   : org.apache.hadoop.io.compress.SplitCompressionInputStream
required: org.apache.hadoop.fs.Seekable
        filePosition = cIn; // take pos from compressed stream
                       ^
org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java(org/apache/hadoop/mapreduce/lib/input:LineRecordReader.java):98:
incompatible types
found   : org.apache.hadoop.io.compress.SplitCompressionInputStream
required: org.apache.hadoop.fs.Seekable
        filePosition = cIn;



I have attached the code 

import java.io.DataInput;  

import java.io.DataOutput;  

import java.io.IOException;  

import java.util.StringTokenizer;  

import org.apache.hadoop.conf.Configured;  

import org.apache.hadoop.fs.FSDataInputStream;  

import org.apache.hadoop.fs.FileSystem;  

import org.apache.hadoop.fs.Path;  

import org.apache.hadoop.io.IntWritable;  

import org.apache.hadoop.io.Text;  

import org.apache.hadoop.io.WritableComparable;  

import org.apache.hadoop.mapreduce.InputSplit;  

import org.apache.hadoop.mapreduce.Job;  

import org.apache.hadoop.mapreduce.Mapper;  

import org.apache.hadoop.mapreduce.RecordReader;  

import org.apache.hadoop.mapreduce.TaskAttemptContext;  

import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;  

import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;  

import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;  

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  

import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;  

import org.apache.hadoop.util.LineReader;  

import org.apache.hadoop.util.Tool;  

import org.apache.hadoop.util.ToolRunner;  


 /**  

  * MultiFileWordCount is an example to demonstrate the usage of   

  * MultiFileInputFormat. This examples counts the occurrences of  

  * words in the text files under the given input directory.  

  */ 

public class MultiFileWordCount extends Configured implements Tool {  

   /**  

    * This record keeps <filename,offset> pairs.  

    */ 

public static class WordOffset implements WritableComparable {  

   private long offset;  

   private String fileName;  
  
   public void readFields(DataInput in) throws IOException {  

      this.offset = in.readLong();  

      this.fileName = Text.readString(in);  

     }  

     public void write(DataOutput out) throws IOException {  

       out.writeLong(offset);  

       Text.writeString(out, fileName);  

     }  

      public int compareTo(Object o) {  

       WordOffset that = (WordOffset)o;  

       int f = this.fileName.compareTo(that.fileName);  

       if(f == 0) {  

         return (int)Math.signum((double)(this.offset - that.offset));  

       }  

       return f;  

     }  

     @Override 

     public boolean equals(Object obj) {  

       if(obj instanceof WordOffset)  

       return this.compareTo(obj) == 0;  

       return false;  

     }  

     @Override 

     public int hashCode() {  

     assert false : "hashCode not designed";  

     return 42; //an arbitrary constant  

     }  

   }  

   /**  

    * To use {@link CombineFileInputFormat}, one should extend it, to return a   

    * (custom) {@link RecordReader}. CombineFileInputFormat uses   

    * {@link CombineFileSplit}s.   

    */ 

   public static class MyInputFormat   

     extends CombineFileInputFormat  {  

     public RecordReader createRecordReader(InputSplit split,  

      TaskAttemptContext context) throws IOException {  

       return new CombineFileRecordReader(  

         (CombineFileSplit)split, context, CombineFileLineRecordReader.class);  

     }  

   }  

  
   /**  

    * RecordReader is responsible from extracting records from a chunk  

    * of the CombineFileSplit.   

    */ 

   public static class CombineFileLineRecordReader   

     extends RecordReader {  

     private long startOffset; //offset of the chunk;  

     private long end; //end of the chunk;  

     private long pos; // current pos   

     private FileSystem fs;  

     private Path path;  

     private WordOffset key;  

     private Text value;  

     private FSDataInputStream fileIn;  

     private LineReader reader;  

     public CombineFileLineRecordReader(CombineFileSplit split,  

         TaskAttemptContext context, Integer index) throws IOException {  
      
       this.path = split.getPath(index);  

       fs = this.path.getFileSystem(context.getConfiguration());  

       this.startOffset = split.getOffset(index);  

       this.end = startOffset + split.getLength(index);  

       boolean skipFirstLine = false;  

       //open the file  

       fileIn = fs.open(path);  

       if (startOffset != 0) {  

       skipFirstLine = true;  

         --startOffset;  

         fileIn.seek(startOffset);  

       }  

       reader = new LineReader(fileIn);  

       if (skipFirstLine) {  // skip first line and re-establish "startOffset".  

        startOffset += reader.readLine(new Text(), 0,  

         (int)Math.min((long)Integer.MAX_VALUE, end - startOffset));  

       }  

       this.pos = startOffset;  

     }  

     public void initialize(InputSplit split, TaskAttemptContext context)  

        throws IOException, InterruptedException {  

     }
     public void close() throws IOException { }  

     public float getProgress() throws IOException {  

       if (startOffset == end) {  

        return 0.0f;  

       } else {  

       return Math.min(1.0f, (pos - startOffset) / (float)(end - startOffset));  

       }  

     }  

    public boolean nextKeyValue() throws IOException {  

       if (key == null) {  

         key = new WordOffset();  

         key.fileName = path.getName();  

       }  

       key.offset = pos;  

       if (value == null) {  

       value = new Text();  

       }  

       int newSize = 0;  

       if (pos < end) {  

        newSize = reader.readLine(value);  

        pos += newSize;  

       }  

       if (newSize == 0) {  

        key = null;  

        value = null;  

        return false;  

       } else {  

        return true;  

       }  

     }  

         public WordOffset getCurrentKey()   

         throws IOException, InterruptedException {  

          return key;  

     }  

         public Text getCurrentValue() throws IOException, InterruptedException {  

         return value;  

     }  

   }  

  
   /**  

    * This Mapper is similar to the one in {@link WordCount.MapClass}.  

    */ 

   public static class MapClass extends  

       Mapper {  

     private final static IntWritable one = new IntWritable(1);  

     private Text word = new Text();  

     public void map(WordOffset key, Text value, Context context)  

         throws IOException, InterruptedException {  

       String line = value.toString();  

       StringTokenizer itr = new StringTokenizer(line);  

       while (itr.hasMoreTokens()) {  

         word.set(itr.nextToken());  

         context.write(word, one);  

       }  

     }  

   }  

   private void printUsage() {  

     System.out.println("Usage : multifilewc  " );  

   }  
   public int run(String[] args) throws Exception {  

     if(args.length < 2) {  

       printUsage();  

       return 2;  

     }  

     Job job = new Job(getConf());  

     job.setJobName("MultiFileWordCount");  

     job.setJarByClass(MultiFileWordCount.class);  
    
     //set the InputFormat of the job to our InputFormat  

     job.setInputFormatClass(MyInputFormat.class);  
       
     // the keys are words (strings)  

     job.setOutputKeyClass(Text.class);  

     // the values are counts (ints)  

     job.setOutputValueClass(IntWritable.class);  
 
     //use the defined mapper  

     job.setMapperClass(MapClass.class);  

     //use the WordCount Reducer  

     job.setCombinerClass(IntSumReducer.class);  

     job.setReducerClass(IntSumReducer.class);  
   

     FileInputFormat.addInputPaths(job, args[0]);  

     FileOutputFormat.setOutputPath(job, new Path(args[1]));  

     return job.waitForCompletion(true) ? 0 : 1;  

   }  
   
   public static void main(String[] args) throws Exception {  

     int ret = ToolRunner.run(new MultiFileWordCount(), args);  

     System.exit(ret);  

   }  

 } 




--
This message was sent by Atlassian JIRA
(v6.1#6144)

Mime
View raw message