hadoop-common-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Ratner, Alan S (IS)" <Alan.Rat...@ngc.com>
Subject setInt & getInt
Date Tue, 04 Oct 2011 16:28:10 GMT
I have no problem with Hadoop.mapred using JobConf to setInt integers and pass them to my map(s)
for getInt as shown in the first program below.  However, when I use Hadoop.mapreduce using
Configuration to setInt these values are invisible to my map's getInt's.  Please tell me what
I am doing wrong.  Thanks.

Both programs expect to see a file with a line or two of text in a directory named testIn.

Alan Ratner


This program uses JobConf and setInt/getInt and works fine.  It outputs:
number = 12345 (from map)


package cbTest;

import java.io.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.*;

public class ConfTest extends Configured implements Tool {

            @SuppressWarnings("deprecation")
            public static class MapClass extends MapReduceBase implements Mapper<LongWritable,
Text, Text, Text> {
                        public static int number;

                        public void configure(JobConf job) {
                                    number = job.getInt("number",-999);
                        }

                        public void map(LongWritable key, Text t, OutputCollector<Text,
Text> output,
                                                Reporter reporter) throws IOException {
                                    System.out.println("number = " + number);
                        }
            }

            @SuppressWarnings("deprecation")
            public int run(String[] args) throws Exception {
                        Path InputDirectory = new Path("testIn");
                        Path OutputDirectory = new Path("testOut");
                        System.out.println(">>>> Running ConfTest Program");
                        JobConf conf = new JobConf(getConf(), ConfTest.class);
                        conf.setInputFormat(TextInputFormat.class);
                        conf.setOutputKeyClass(Text.class);
                        conf.setOutputValueClass(Text.class);
                        conf.setMapperClass(MapClass.class);
                        conf.setInt("number", 12345);
                        FileInputFormat.setInputPaths(conf, InputDirectory);
                        FileOutputFormat.setOutputPath(conf, OutputDirectory);
                        FileSystem fs = OutputDirectory.getFileSystem(conf);
                        fs.delete(OutputDirectory, true); //remove output of prior run
                        JobClient.runJob(conf);
                        return 0;
            }

            public static void main(String[] args) throws Exception {
                        int res = ToolRunner.run(new Configuration(), new ConfTest(), args);
                        System.exit(res);
            }
}


This program uses Configuration and setInt/getInt.  Butt getInt in neither map or map:configure
works.  It outputs:
>>>>> Passing integer 12345 in configuration <<<<< (from run)
map numbers: -999 -1 (from map as intMapConf and intConfConf)


package cbTest;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class Conf2Test extends Configured implements Tool
{
            public static class MapClass extends Mapper<LongWritable, Text, Text, Text>
    {
                        public int intConfConf = -1;

                        public void configure(Configuration job) {
                                    intConfConf = job.getInt("number", -2);
                        }

        public void map(LongWritable key, Text value, Context context) throws IOException,
InterruptedException
        {
                        int intMapConf = context.getConfiguration().getInt("number", -999);
                        System.out.println("map numbers: " + intMapConf+" "+intConfConf);
        }
    }

    public static void main(String[] args) throws Exception
    {
        int res = ToolRunner.run(new Configuration(), new Conf2Test(), args);
        System.exit(res);

    }

            public int run(String[] arg0) throws Exception {
                        Path Input_Directory = new Path("testIn");
                        Path Output_Directory = new Path("testOut");

                        Configuration conf = new Configuration();
                        Job job = new Job(conf, Conf2Test.class.getSimpleName());
                        job.setJarByClass(Conf2Test.class);
                        job.setMapperClass(MapClass.class);
                        job.setMapOutputKeyClass(Text.class);
                        job.setMapOutputValueClass(Text.class);
                        TextOutputFormat.setOutputPath(job, Output_Directory);
                        TextInputFormat.setInputPaths(job, Input_Directory);
                        FileSystem hdfs = FileSystem.get(conf);
                        hdfs.delete(Output_Directory, true);

                        conf.setInt("number", 12345);
                        System.out.println(">>>>> Passing integer "+conf.getInt("number",
-1)+" in configuration <<<<<");

                        job.waitForCompletion(true);
                        return 0;
            }
}


Alan Ratner

Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message