hadoop-mapreduce-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Jason Lowe (JIRA)" <j...@apache.org>
Subject [jira] [Resolved] (MAPREDUCE-5666) org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):cannot find symbol
Date Wed, 04 Dec 2013 14:36:36 GMT

     [ https://issues.apache.org/jira/browse/MAPREDUCE-5666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Jason Lowe resolved MAPREDUCE-5666.
-----------------------------------

    Resolution: Invalid

As I mentioned in MAPREDUCE-5668, JIRA is not the avenue to use for questions like this. 
Please use the [user@ mailing list|http://hadoop.apache.org/mailing_lists.html#User] for questions
like this.  If after discussing on the mailing list it ends up being a bug in Hadoop then
a JIRA can be filed at that time.

This is clearly a case of code being compiled against a release after 1.x but then run on
a 0.20 or 1.x release.  The FileStatus.isDirectory() method was not present in the 0.20 or
1.x releases.  We cannot generally support compiling code against a later release and then
running it on an earlier release because of new APIs that can be added.

> org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):cannot
find symbol
> -----------------------------------------------------------------------------------------------------------------------------------------
>
>                 Key: MAPREDUCE-5666
>                 URL: https://issues.apache.org/jira/browse/MAPREDUCE-5666
>             Project: Hadoop Map/Reduce
>          Issue Type: Bug
>            Reporter: ranjini
>
> hi 
> I have written the below code , and facing the issue. i am using hadoop 0.20 vesion and
java 1.6 the issue is 
> org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java(org/apache/hadoop/mapreduce/lib/input:FileInputFormat.java):232:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>           if (globStat.isDirectory()) {
>                       ^
> org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java(org/apache/hadoop/mapreduce/lib/output:FileOutputCommitter.java):208:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>     } else if(fs.getFileStatus(taskOutput).isDirectory()) {
>                                           ^
> org/apache/hadoop/mapred/JobConf.java(org/apache/hadoop/mapred:JobConf.java):433: cannot
find symbol
> symbol  : method getPattern(java.lang.String,java.util.regex.Pattern)
> location: class org.apache.hadoop.mapred.JobConf
>     return getPattern(JobContext.JAR_UNPACK_PATTERN, UNPACK_JAR_PATTERN_DEFAULT);
>            ^
> org/apache/hadoop/mapred/JobConf.java(org/apache/hadoop/mapred:JobConf.java):450: cannot
find symbol
> symbol  : method getTrimmedStrings(java.lang.String)
> location: class org.apache.hadoop.mapred.JobConf
>     return getTrimmedStrings(MRConfig.LOCAL_DIR);
>            ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):165:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>       if (stat.isDirectory()) {
>               ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):215:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>           if (globStat.isDirectory()) {
>                       ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):218:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>               if (recursive && stat.isDirectory()) {
>                                    ^
> org/apache/hadoop/mapred/FileInputFormat.java(org/apache/hadoop/mapred:FileInputFormat.java):258:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>       if (file.isDirectory()) {
>               ^
> org/apache/hadoop/mapred/FileOutputCommitter.java(org/apache/hadoop/mapred:FileOutputCommitter.java):166:
cannot find symbol
> symbol  : method isDirectory()
> location: class org.apache.hadoop.fs.FileStatus
>     } else if(fs.getFileStatus(taskOutput).isDirectory()) {
>                                           ^
> org/apache/hadoop/mapred/LineRecordReader.java(org/apache/hadoop/mapred:LineRecordReader.java):100:
incompatible types
> found   : org.apache.hadoop.io.compress.SplitCompressionInputStream
> required: org.apache.hadoop.fs.Seekable
>         filePosition = cIn; // take pos from compressed stream
>                        ^
> org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java(org/apache/hadoop/mapreduce/lib/input:LineRecordReader.java):98:
incompatible types
> found   : org.apache.hadoop.io.compress.SplitCompressionInputStream
> required: org.apache.hadoop.fs.Seekable
>         filePosition = cIn;
> I have attached the code 
> import java.io.DataInput;  
> import java.io.DataOutput;  
> import java.io.IOException;  
> import java.util.StringTokenizer;  
> import org.apache.hadoop.conf.Configured;  
> import org.apache.hadoop.fs.FSDataInputStream;  
> import org.apache.hadoop.fs.FileSystem;  
> import org.apache.hadoop.fs.Path;  
> import org.apache.hadoop.io.IntWritable;  
> import org.apache.hadoop.io.Text;  
> import org.apache.hadoop.io.WritableComparable;  
> import org.apache.hadoop.mapreduce.InputSplit;  
> import org.apache.hadoop.mapreduce.Job;  
> import org.apache.hadoop.mapreduce.Mapper;  
> import org.apache.hadoop.mapreduce.RecordReader;  
> import org.apache.hadoop.mapreduce.TaskAttemptContext;  
> import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;  
> import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;  
> import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;  
> import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
> import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
> import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;  
> import org.apache.hadoop.util.LineReader;  
> import org.apache.hadoop.util.Tool;  
> import org.apache.hadoop.util.ToolRunner;  
>  /**  
>   * MultiFileWordCount is an example to demonstrate the usage of   
>   * MultiFileInputFormat. This examples counts the occurrences of  
>   * words in the text files under the given input directory.  
>   */ 
> public class MultiFileWordCount extends Configured implements Tool {  
>    /**  
>     * This record keeps <filename,offset> pairs.  
>     */ 
> public static class WordOffset implements WritableComparable {  
>    private long offset;  
>    private String fileName;  
>   
>    public void readFields(DataInput in) throws IOException {  
>       this.offset = in.readLong();  
>       this.fileName = Text.readString(in);  
>      }  
>      public void write(DataOutput out) throws IOException {  
>        out.writeLong(offset);  
>        Text.writeString(out, fileName);  
>      }  
>       public int compareTo(Object o) {  
>        WordOffset that = (WordOffset)o;  
>        int f = this.fileName.compareTo(that.fileName);  
>        if(f == 0) {  
>          return (int)Math.signum((double)(this.offset - that.offset));  
>        }  
>        return f;  
>      }  
>      @Override 
>      public boolean equals(Object obj) {  
>        if(obj instanceof WordOffset)  
>        return this.compareTo(obj) == 0;  
>        return false;  
>      }  
>      @Override 
>      public int hashCode() {  
>      assert false : "hashCode not designed";  
>      return 42; //an arbitrary constant  
>      }  
>    }  
>    /**  
>     * To use {@link CombineFileInputFormat}, one should extend it, to return a   
>     * (custom) {@link RecordReader}. CombineFileInputFormat uses   
>     * {@link CombineFileSplit}s.   
>     */ 
>    public static class MyInputFormat   
>      extends CombineFileInputFormat  {  
>      public RecordReader createRecordReader(InputSplit split,  
>       TaskAttemptContext context) throws IOException {  
>        return new CombineFileRecordReader(  
>          (CombineFileSplit)split, context, CombineFileLineRecordReader.class);  
>      }  
>    }  
>   
>    /**  
>     * RecordReader is responsible from extracting records from a chunk  
>     * of the CombineFileSplit.   
>     */ 
>    public static class CombineFileLineRecordReader   
>      extends RecordReader {  
>      private long startOffset; //offset of the chunk;  
>      private long end; //end of the chunk;  
>      private long pos; // current pos   
>      private FileSystem fs;  
>      private Path path;  
>      private WordOffset key;  
>      private Text value;  
>      private FSDataInputStream fileIn;  
>      private LineReader reader;  
>      public CombineFileLineRecordReader(CombineFileSplit split,  
>          TaskAttemptContext context, Integer index) throws IOException {  
>       
>        this.path = split.getPath(index);  
>        fs = this.path.getFileSystem(context.getConfiguration());  
>        this.startOffset = split.getOffset(index);  
>        this.end = startOffset + split.getLength(index);  
>        boolean skipFirstLine = false;  
>        //open the file  
>        fileIn = fs.open(path);  
>        if (startOffset != 0) {  
>        skipFirstLine = true;  
>          --startOffset;  
>          fileIn.seek(startOffset);  
>        }  
>        reader = new LineReader(fileIn);  
>        if (skipFirstLine) {  // skip first line and re-establish "startOffset".  
>         startOffset += reader.readLine(new Text(), 0,  
>          (int)Math.min((long)Integer.MAX_VALUE, end - startOffset));  
>        }  
>        this.pos = startOffset;  
>      }  
>      public void initialize(InputSplit split, TaskAttemptContext context)  
>         throws IOException, InterruptedException {  
>      }
>      public void close() throws IOException { }  
>      public float getProgress() throws IOException {  
>        if (startOffset == end) {  
>         return 0.0f;  
>        } else {  
>        return Math.min(1.0f, (pos - startOffset) / (float)(end - startOffset));  
>        }  
>      }  
>     public boolean nextKeyValue() throws IOException {  
>        if (key == null) {  
>          key = new WordOffset();  
>          key.fileName = path.getName();  
>        }  
>        key.offset = pos;  
>        if (value == null) {  
>        value = new Text();  
>        }  
>        int newSize = 0;  
>        if (pos < end) {  
>         newSize = reader.readLine(value);  
>         pos += newSize;  
>        }  
>        if (newSize == 0) {  
>         key = null;  
>         value = null;  
>         return false;  
>        } else {  
>         return true;  
>        }  
>      }  
>          public WordOffset getCurrentKey()   
>          throws IOException, InterruptedException {  
>           return key;  
>      }  
>          public Text getCurrentValue() throws IOException, InterruptedException {  
>          return value;  
>      }  
>    }  
>   
>    /**  
>     * This Mapper is similar to the one in {@link WordCount.MapClass}.  
>     */ 
>    public static class MapClass extends  
>        Mapper {  
>      private final static IntWritable one = new IntWritable(1);  
>      private Text word = new Text();  
>      public void map(WordOffset key, Text value, Context context)  
>          throws IOException, InterruptedException {  
>        String line = value.toString();  
>        StringTokenizer itr = new StringTokenizer(line);  
>        while (itr.hasMoreTokens()) {  
>          word.set(itr.nextToken());  
>          context.write(word, one);  
>        }  
>      }  
>    }  
>    private void printUsage() {  
>      System.out.println("Usage : multifilewc  " );  
>    }  
>    public int run(String[] args) throws Exception {  
>      if(args.length < 2) {  
>        printUsage();  
>        return 2;  
>      }  
>      Job job = new Job(getConf());  
>      job.setJobName("MultiFileWordCount");  
>      job.setJarByClass(MultiFileWordCount.class);  
>     
>      //set the InputFormat of the job to our InputFormat  
>      job.setInputFormatClass(MyInputFormat.class);  
>        
>      // the keys are words (strings)  
>      job.setOutputKeyClass(Text.class);  
>      // the values are counts (ints)  
>      job.setOutputValueClass(IntWritable.class);  
>  
>      //use the defined mapper  
>      job.setMapperClass(MapClass.class);  
>      //use the WordCount Reducer  
>      job.setCombinerClass(IntSumReducer.class);  
>      job.setReducerClass(IntSumReducer.class);  
>    
>      FileInputFormat.addInputPaths(job, args[0]);  
>      FileOutputFormat.setOutputPath(job, new Path(args[1]));  
>      return job.waitForCompletion(true) ? 0 : 1;  
>    }  
>    
>    public static void main(String[] args) throws Exception {  
>      int ret = ToolRunner.run(new MultiFileWordCount(), args);  
>      System.exit(ret);  
>    }  
>  } 



--
This message was sent by Atlassian JIRA
(v6.1#6144)

Mime
View raw message