Return-Path: Delivered-To: apmail-lucene-hadoop-commits-archive@locus.apache.org Received: (qmail 96844 invoked from network); 30 Mar 2006 22:17:51 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (209.237.227.199) by minotaur.apache.org with SMTP; 30 Mar 2006 22:17:51 -0000 Received: (qmail 77328 invoked by uid 500); 30 Mar 2006 22:17:51 -0000 Delivered-To: apmail-lucene-hadoop-commits-archive@lucene.apache.org Received: (qmail 77299 invoked by uid 500); 30 Mar 2006 22:17:51 -0000 Mailing-List: contact hadoop-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hadoop-dev@lucene.apache.org Delivered-To: mailing list hadoop-commits@lucene.apache.org Received: (qmail 77288 invoked by uid 99); 30 Mar 2006 22:17:51 -0000 Received: from asf.osuosl.org (HELO asf.osuosl.org) (140.211.166.49) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Mar 2006 14:17:51 -0800 X-ASF-Spam-Status: No, hits=-9.4 required=10.0 tests=ALL_TRUSTED,NO_REAL_NAME X-Spam-Check-By: apache.org Received: from [209.237.227.194] (HELO minotaur.apache.org) (209.237.227.194) by apache.org (qpsmtpd/0.29) with SMTP; Thu, 30 Mar 2006 14:17:50 -0800 Received: (qmail 96665 invoked by uid 65534); 30 Mar 2006 22:17:29 -0000 Message-ID: <20060330221729.96664.qmail@minotaur.apache.org> Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r390258 - in /lucene/hadoop/trunk/src: examples/org/apache/hadoop/examples/ java/org/apache/hadoop/mapred/lib/ Date: Thu, 30 Mar 2006 22:17:28 -0000 To: hadoop-commits@lucene.apache.org From: cutting@apache.org X-Mailer: svnmailer-1.0.7 X-Virus-Checked: Checked by ClamAV on apache.org X-Spam-Rating: minotaur.apache.org 1.6.2 0/1000/N Author: cutting Date: Thu Mar 30 14:17:26 2006 New Revision: 390258 URL: http://svn.apache.org/viewcvs?rev=390258&view=rev Log: Fix for HADOOP-103. Add a base class for Mapper and Reducer implementations that implements Closeable and JobConfigurable. Use it in supplied Mappers & Reducers. Also some minor improvements to demos. Contributed by Owen O'Malley. Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityMapper.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityReducer.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/InverseMapper.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/LongSumReducer.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/RegexMapper.java lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/TokenCountMapper.java Modified: lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java (original) +++ lucene/hadoop/trunk/src/examples/org/apache/hadoop/examples/WordCount.java Thu Mar 30 14:17:26 2006 @@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; /** * This is an example Hadoop Map/Reduce application. @@ -49,9 +50,10 @@ * For each line of input, break the line into words and emit them as * (word, 1). */ - public static class MapClass implements Mapper { + public static class MapClass extends MapReduceBase implements Mapper { private final static IntWritable one = new IntWritable(1); + private UTF8 word = new UTF8(); public void map(WritableComparable key, Writable value, OutputCollector output, @@ -59,23 +61,16 @@ String line = ((UTF8)value).toString(); StringTokenizer itr = new StringTokenizer(line); while (itr.hasMoreTokens()) { - String word = itr.nextToken(); - output.collect(new UTF8(word), one); + word.set(itr.nextToken()); + output.collect(word, one); } } - - public void configure(JobConf job) { - } - - public void close() { - } - } /** * A reducer class that just emits the sum of the input values. */ - public static class Reduce implements Reducer { + public static class Reduce extends MapReduceBase implements Reducer { public void reduce(WritableComparable key, Iterator values, OutputCollector output, @@ -86,13 +81,6 @@ } output.collect(key, new IntWritable(sum)); } - - public void configure(JobConf job) { - } - - public void close() { - } - } static void printUsage() { @@ -150,7 +138,7 @@ conf.setOutputDir(new File((String) other_args.get(1))); // Uncomment to run locally in a single process - // countJob.set("mapred.job.tracker", "local"); + // conf.set("mapred.job.tracker", "local"); JobClient.runJob(conf); } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityMapper.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityMapper.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityMapper.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityMapper.java Thu Mar 30 14:17:26 2006 @@ -20,16 +20,14 @@ import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; /** Implements the identity function, mapping inputs directly to outputs. */ -public class IdentityMapper implements Mapper { - - public void configure(JobConf job) {} +public class IdentityMapper extends MapReduceBase implements Mapper { /** The identify function. Input key/value pair is written directly to * output.*/ @@ -38,5 +36,4 @@ throws IOException { output.collect(key, val); } - public void close() {} } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityReducer.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityReducer.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityReducer.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/IdentityReducer.java Thu Mar 30 14:17:26 2006 @@ -22,16 +22,14 @@ import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; /** Performs no reduction, writing all input values directly to the output. */ -public class IdentityReducer implements Reducer { - - public void configure(JobConf job) {} +public class IdentityReducer extends MapReduceBase implements Reducer { /** Writes all keys and values directly to output. */ public void reduce(WritableComparable key, Iterator values, @@ -41,7 +39,5 @@ output.collect(key, (Writable)values.next()); } } - - public void close() {} } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/InverseMapper.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/InverseMapper.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/InverseMapper.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/InverseMapper.java Thu Mar 30 14:17:26 2006 @@ -20,17 +20,15 @@ import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.Writable; /** A {@link Mapper} that swaps keys and values. */ -public class InverseMapper implements Mapper { - - public void configure(JobConf job) {} +public class InverseMapper extends MapReduceBase implements Mapper { /** The inverse function. Input keys and values are swapped.*/ public void map(WritableComparable key, Writable value, @@ -38,7 +36,5 @@ throws IOException { output.collect((WritableComparable)value, key); } - - public void close() {} } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/LongSumReducer.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/LongSumReducer.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/LongSumReducer.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/LongSumReducer.java Thu Mar 30 14:17:26 2006 @@ -21,16 +21,14 @@ import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.LongWritable; /** A {@link Reducer} that sums long values. */ -public class LongSumReducer implements Reducer { - - public void configure(JobConf job) {} +public class LongSumReducer extends MapReduceBase implements Reducer { public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter) @@ -45,7 +43,5 @@ // output sum output.collect(key, new LongWritable(sum)); } - - public void close() {} - + } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/RegexMapper.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/RegexMapper.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/RegexMapper.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/RegexMapper.java Thu Mar 30 14:17:26 2006 @@ -22,6 +22,7 @@ import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.Writable; @@ -34,7 +35,7 @@ /** A {@link Mapper} that extracts text matching a regular expression. */ -public class RegexMapper implements Mapper { +public class RegexMapper extends MapReduceBase implements Mapper { private Pattern pattern; private int group; @@ -53,7 +54,5 @@ output.collect(new UTF8(matcher.group(group)), new LongWritable(1)); } } - - public void close() {} - + } Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/TokenCountMapper.java URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/TokenCountMapper.java?rev=390258&r1=390257&r2=390258&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/TokenCountMapper.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/lib/TokenCountMapper.java Thu Mar 30 14:17:26 2006 @@ -21,8 +21,8 @@ import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.Writable; @@ -32,15 +32,12 @@ /** A {@link Mapper} that maps text values into pairs. Uses * {@link StringTokenizer} to break text into tokens. */ -public class TokenCountMapper implements Mapper { - - public void configure(JobConf job) {} +public class TokenCountMapper extends MapReduceBase implements Mapper { public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter) throws IOException { // get input text - long position = ((LongWritable)key).get(); // key is position in file String text = ((UTF8)value).toString(); // value is line of text // tokenize the value @@ -50,7 +47,5 @@ output.collect(new UTF8(st.nextToken()), new LongWritable(1)); } } - - public void close() {} }