mahout-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sro...@apache.org
Subject svn commit: r885782 - in /lucene/mahout/trunk: core/ core/src/main/java/org/apache/mahout/cf/taste/hadoop/ taste-web/
Date Tue, 01 Dec 2009 15:02:26 GMT
Author: srowen
Date: Tue Dec  1 15:02:25 2009
New Revision: 885782

URL: http://svn.apache.org/viewvc?rev=885782&view=rev
Log:
MAHOUT-143: move CF Hadoop jobs back to hold Hadoop APIs; use most stable official 0.20.x
Hadoop code

Modified:
    lucene/mahout/trunk/core/pom.xml
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/IdentityReducer.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsMapper.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java
    lucene/mahout/trunk/taste-web/pom.xml

Modified: lucene/mahout/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/pom.xml?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/pom.xml (original)
+++ lucene/mahout/trunk/core/pom.xml Tue Dec  1 15:02:25 2009
@@ -177,9 +177,9 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.mahout.hadoop</groupId>
+      <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-core</artifactId>
-      <version>0.20.1</version>
+      <version>0.20.2-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>org.apache.mahout.hbase</groupId>

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/IdentityReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/IdentityReducer.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/IdentityReducer.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/IdentityReducer.java
Tue Dec  1 15:02:25 2009
@@ -18,18 +18,24 @@
 
 package org.apache.mahout.cf.taste.hadoop;
 
-import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
 
 import java.io.IOException;
+import java.util.Iterator;
 
 /** Copied from Hadoop 0.19. Replace when Hadoop 0.20+ makes Reducer non-abstract. */
-public class IdentityReducer<K, V> extends Reducer<K, V, K, V> {
+public class IdentityReducer<K, V> extends MapReduceBase implements Reducer<K, V,
K, V> {
 
   @Override
-  protected void reduce(K key, Iterable<V> values, Context context
-  ) throws IOException, InterruptedException {
-    for (V value : values) {
-      context.write(key, value);
+  public void reduce(K key,
+                     Iterator<V> values,
+                     OutputCollector<K, V> output,
+                     Reporter reporter) throws IOException {
+    while (values.hasNext()) {
+      output.collect(key, values.next());
     }
   }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderJob.java
Tue Dec  1 15:02:25 2009
@@ -25,15 +25,17 @@
 import org.apache.commons.cli2.builder.DefaultOptionBuilder;
 import org.apache.commons.cli2.builder.GroupBuilder;
 import org.apache.commons.cli2.commandline.Parser;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.mahout.cf.taste.recommender.Recommender;
 import org.apache.mahout.common.CommandLineUtil;
@@ -105,12 +107,11 @@
  *   10 input/users.txt input/input.csv recommender.jar output
  * }
  */
-public final class RecommenderJob extends Job {
+public final class RecommenderJob {
 
   private static final Logger log = LoggerFactory.getLogger(RecommenderJob.class);
 
-  public RecommenderJob(Configuration jobConf) throws IOException {
-    super(jobConf);
+  private RecommenderJob() {
   }
 
   public static void main(String[] args) throws Exception {
@@ -161,24 +162,23 @@
       String dataModelFile = cmdLine.getValue(dataModelFileOpt).toString();
       String jarFile = cmdLine.getValue(jarFileOpt).toString();
       String outputPath = cmdLine.getValue(outputOpt).toString();
-      Configuration jobConf =
+      JobConf jobConf =
           buildJobConf(recommendClassName, recommendationsPerUser, userIDFile, dataModelFile,
jarFile, outputPath);
-      Job job = new RecommenderJob(jobConf);
-      job.waitForCompletion(true); 
+      JobClient.runJob(jobConf);
     } catch (OptionException e) {
       log.error(e.getMessage());
       CommandLineUtil.printHelp(group);
     }
   }
 
-  public static Configuration buildJobConf(String recommendClassName,
-                                           int recommendationsPerUser,
-                                           String userIDFile,
-                                           String dataModelFile,
-                                           String jarFile,
-                                           String outputPath) throws IOException {
+  public static JobConf buildJobConf(String recommendClassName,
+                                     int recommendationsPerUser,
+                                     String userIDFile,
+                                     String dataModelFile,
+                                     String jarFile,
+                                     String outputPath) throws IOException {
 
-    Configuration jobConf = new Configuration();
+    JobConf jobConf = new JobConf();
     FileSystem fs = FileSystem.get(jobConf);
 
     Path userIDFilePath = new Path(userIDFile).makeQualified(fs);
@@ -189,6 +189,7 @@
     }
 
     jobConf.set("mapred.jar", jarFile);
+    jobConf.setJar(jarFile);
 
     jobConf.set(RecommenderMapper.RECOMMENDER_CLASS_NAME, recommendClassName);
     jobConf.set(RecommenderMapper.RECOMMENDATIONS_PER_USER, String.valueOf(recommendationsPerUser));
@@ -205,7 +206,7 @@
     jobConf.setClass("mapred.output.key.class", LongWritable.class, Object.class);
     jobConf.setClass("mapred.output.value.class", RecommendedItemsWritable.class, Object.class);
 
-    //jobConf.setClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class);
+    jobConf.setClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class);
     jobConf.set("mapred.output.dir", StringUtils.escapeString(outputPathPath.toString()));
 
     return jobConf;

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/RecommenderMapper.java
Tue Dec  1 15:02:25 2009
@@ -17,11 +17,13 @@
 
 package org.apache.mahout.cf.taste.hadoop;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
 import org.apache.mahout.cf.taste.common.TasteException;
 import org.apache.mahout.cf.taste.impl.model.file.FileDataModel;
 import org.apache.mahout.cf.taste.model.DataModel;
@@ -46,7 +48,8 @@
  * @see RecommenderJob
  */
 public final class RecommenderMapper
-    extends Mapper<LongWritable, LongWritable, LongWritable, RecommendedItemsWritable>
{
+    extends MapReduceBase
+    implements Mapper<LongWritable, LongWritable, LongWritable, RecommendedItemsWritable>
{
 
   static final String RECOMMENDER_CLASS_NAME = "recommenderClassName";
   static final String RECOMMENDATIONS_PER_USER = "recommendationsPerUser";
@@ -56,24 +59,7 @@
   private int recommendationsPerUser;
 
   @Override
-  protected void map(LongWritable key, LongWritable value,
-                     Context context) throws IOException, InterruptedException {
-    long userID = value.get();
-    List<RecommendedItem> recommendedItems;
-    try {
-      recommendedItems = recommender.recommend(userID, recommendationsPerUser);
-    } catch (TasteException te) {
-      throw new IllegalStateException(te);
-    }
-    RecommendedItemsWritable writable = new RecommendedItemsWritable(recommendedItems);
-    context.write(value, writable);
-    context.getCounter(ReducerMetrics.USERS_PROCESSED).increment(1L);
-    context.getCounter(ReducerMetrics.RECOMMENDATIONS_MADE).increment(recommendedItems.size());
-  }
-
-  @Override
-  protected void setup(Context context) {
-    Configuration jobConf = context.getConfiguration();
+  public void configure(org.apache.hadoop.mapred.JobConf jobConf) {
     String dataModelFile = jobConf.get(DATA_MODEL_FILE);
     String recommenderClassName = jobConf.get(RECOMMENDER_CLASS_NAME);
     FileDataModel fileDataModel;
@@ -107,4 +93,22 @@
     recommendationsPerUser = Integer.parseInt(jobConf.get(RECOMMENDATIONS_PER_USER));
   }
 
+
+  @Override
+  public void map(LongWritable key,
+                  LongWritable value,
+                  OutputCollector<LongWritable, RecommendedItemsWritable> output,
+                  Reporter reporter) throws IOException {
+    long userID = value.get();
+    List<RecommendedItem> recommendedItems;
+    try {
+      recommendedItems = recommender.recommend(userID, recommendationsPerUser);
+    } catch (TasteException te) {
+      throw new IllegalStateException(te);
+    }
+    RecommendedItemsWritable writable = new RecommendedItemsWritable(recommendedItems);
+    output.collect(value, writable);
+    reporter.getCounter(ReducerMetrics.USERS_PROCESSED).increment(1L);
+    reporter.getCounter(ReducerMetrics.RECOMMENDATIONS_MADE).increment(recommendedItems.size());
+  }
 }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java
Tue Dec  1 15:02:25 2009
@@ -25,17 +25,17 @@
 import org.apache.commons.cli2.builder.DefaultOptionBuilder;
 import org.apache.commons.cli2.builder.GroupBuilder;
 import org.apache.commons.cli2.commandline.Parser;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.mahout.common.CommandLineUtil;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
@@ -44,13 +44,12 @@
 
 import java.io.IOException;
 
-public final class SlopeOneDiffsToAveragesJob extends Job {
+public final class SlopeOneDiffsToAveragesJob {
 
   /** Logger for this class. */
   private static final Logger log = LoggerFactory.getLogger(SlopeOneDiffsToAveragesJob.class);
 
-  private SlopeOneDiffsToAveragesJob(Configuration jobConf) throws IOException {
-    super(jobConf);
+  private SlopeOneDiffsToAveragesJob() {
   }
 
   public static void main(String[] args) throws Exception {
@@ -64,8 +63,12 @@
     Option outputOpt = DefaultOptionCreator.outputOption(obuilder, abuilder).create();
     Option helpOpt = DefaultOptionCreator.helpOption(obuilder);
 
+    Option jarFileOpt = obuilder.withLongName("jarFile").withRequired(true)
+      .withShortName("m").withArgument(abuilder.withName("jarFile").withMinimum(1)
+      .withMaximum(1).create()).withDescription("Implementation jar.").create();
+
     Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt)
-        .withOption(helpOpt).create();
+        .withOption(jarFileOpt).withOption(helpOpt).create();
 
 
     try {
@@ -80,19 +83,20 @@
 
       String prefsFile = cmdLine.getValue(inputOpt).toString();
       String outputPath = cmdLine.getValue(outputOpt).toString();
-      Configuration jobConf = buildJobConf(prefsFile, outputPath);
-      Job job = new SlopeOneDiffsToAveragesJob(jobConf);
-      job.waitForCompletion(true);
+      String jarFile = cmdLine.getValue(jarFileOpt).toString();
+      JobConf jobConf = buildJobConf(prefsFile, outputPath, jarFile);
+      JobClient.runJob(jobConf);
     } catch (OptionException e) {
       log.error(e.getMessage());
       CommandLineUtil.printHelp(group);
     }
   }
 
-  public static Configuration buildJobConf(String prefsFile,
-                                           String outputPath) throws IOException {
+  public static JobConf buildJobConf(String prefsFile,
+                                     String outputPath,
+                                     String jarFile) throws IOException {
 
-    Configuration jobConf = new Configuration();
+    JobConf jobConf = new JobConf();
     FileSystem fs = FileSystem.get(jobConf);
 
     Path prefsFilePath = new Path(prefsFile).makeQualified(fs);
@@ -102,6 +106,9 @@
       fs.delete(outputPathPath, true);
     }
 
+    jobConf.set("mapred.jar", jarFile);
+    jobConf.setJar(jarFile);
+
     jobConf.setClass("mapred.input.format.class", SequenceFileInputFormat.class, InputFormat.class);
     jobConf.set("mapred.input.dir", StringUtils.escapeString(prefsFilePath.toString()));
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java
Tue Dec  1 15:02:25 2009
@@ -18,23 +18,29 @@
 package org.apache.mahout.cf.taste.hadoop;
 
 import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
 
 import java.io.IOException;
+import java.util.Iterator;
 
 public final class SlopeOneDiffsToAveragesReducer
-    extends Reducer<ItemItemWritable, FloatWritable, ItemItemWritable, FloatWritable>
{
+    extends MapReduceBase
+    implements Reducer<ItemItemWritable, FloatWritable, ItemItemWritable, FloatWritable>
{
 
   @Override
-  protected void reduce(ItemItemWritable key, Iterable<FloatWritable> values, Context
context)
-      throws IOException, InterruptedException {
+  public void reduce(ItemItemWritable key,
+                     Iterator<FloatWritable> values,
+                     OutputCollector<ItemItemWritable, FloatWritable> output,
+                     Reporter reporter) throws IOException {
     int count = 0;
     double total = 0.0;
-    for (FloatWritable value : values) {
-      total += value.get();
+    while (values.hasNext()) {
+      total += values.next().get();
       count++;
     }
-    context.write(key, new FloatWritable((float) (total / count)));
+    output.collect(key, new FloatWritable((float) (total / count)));
   }
-
 }
\ No newline at end of file

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java
Tue Dec  1 15:02:25 2009
@@ -30,13 +30,14 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.mahout.common.CommandLineUtil;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
@@ -45,13 +46,12 @@
 
 import java.io.IOException;
 
-public final class SlopeOnePrefsToDiffsJob extends Job {
+public final class SlopeOnePrefsToDiffsJob {
 
   /** Logger for this class.*/
   private static final Logger log = LoggerFactory.getLogger(SlopeOnePrefsToDiffsJob.class);
 
-  private SlopeOnePrefsToDiffsJob(Configuration jobConf) throws IOException {
-    super(jobConf);
+  private SlopeOnePrefsToDiffsJob() throws IOException {
   }
 
   public static void main(String[] args) throws Exception {
@@ -63,11 +63,15 @@
     .withArgument(abuilder.withName("input").withMinimum(1).withMaximum(1).create())
     .withDescription("The Path for input preferences file.").create();
 
+    Option jarFileOpt = obuilder.withLongName("jarFile").withRequired(true)
+      .withShortName("m").withArgument(abuilder.withName("jarFile").withMinimum(1)
+      .withMaximum(1).create()).withDescription("Implementation jar.").create();
+
     Option outputOpt = DefaultOptionCreator.outputOption(obuilder, abuilder).create();
     Option helpOpt = DefaultOptionCreator.helpOption(obuilder);
 
     Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt)
-        .withOption(helpOpt).create();
+        .withOption(jarFileOpt).withOption(helpOpt).create();
 
 
     try {
@@ -82,19 +86,20 @@
 
       String prefsFile = cmdLine.getValue(inputOpt).toString();
       String outputPath = cmdLine.getValue(outputOpt).toString();
-      Configuration jobConf = buildJobConf(prefsFile, outputPath);
-      Job job = new SlopeOnePrefsToDiffsJob(jobConf);
-      job.waitForCompletion(true);
+      String jarFile = cmdLine.getValue(jarFileOpt).toString();
+      JobConf jobConf = buildJobConf(prefsFile, outputPath, jarFile);
+      JobClient.runJob(jobConf);
     } catch (OptionException e) {
       log.error(e.getMessage());
       CommandLineUtil.printHelp(group);
     }
   }
 
-  public static Configuration buildJobConf(String prefsFile,
-                                           String outputPath) throws IOException {
+  public static JobConf buildJobConf(String prefsFile,
+                                     String outputPath,
+                                     String jarFile) throws IOException {
 
-    Configuration jobConf = new Configuration();
+    JobConf jobConf = new JobConf();
     FileSystem fs = FileSystem.get(jobConf);
 
     Path prefsFilePath = new Path(prefsFile).makeQualified(fs);
@@ -104,6 +109,9 @@
       fs.delete(outputPathPath, true);
     }
 
+    jobConf.set("mapred.jar", jarFile);
+    jobConf.setJar(jarFile);
+
     jobConf.setClass("mapred.input.format.class", TextInputFormat.class, InputFormat.class);
     jobConf.set("mapred.input.dir", StringUtils.escapeString(prefsFilePath.toString()));
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsMapper.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsMapper.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsMapper.java
Tue Dec  1 15:02:25 2009
@@ -19,25 +19,30 @@
 
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
 
 import java.io.IOException;
 import java.util.regex.Pattern;
 
 public final class SlopeOnePrefsToDiffsMapper
-    extends Mapper<LongWritable, Text, LongWritable, ItemPrefWritable> {
+    extends MapReduceBase
+    implements Mapper<LongWritable, Text, LongWritable, ItemPrefWritable> {
 
   private static final Pattern COMMA = Pattern.compile(",");
 
   @Override
-  protected void map(LongWritable key, Text value,
-                     Context context) throws IOException, InterruptedException {
-    String line = value.toString();
-    String[] tokens = COMMA.split(line);
+  public void map(LongWritable key,
+                  Text value,
+                  OutputCollector<LongWritable, ItemPrefWritable> output,
+                  Reporter reporter) throws IOException {
+    String[] tokens = COMMA.split(value.toString());
     long userID = Long.parseLong(tokens[0]);
     long itemID = Long.parseLong(tokens[1]);
     float prefValue = Float.parseFloat(tokens[2]);
-    context.write(new LongWritable(userID), new ItemPrefWritable(itemID, prefValue));
+    output.collect(new LongWritable(userID), new ItemPrefWritable(itemID, prefValue));
   }
 
 }
\ No newline at end of file

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java
(original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java
Tue Dec  1 15:02:25 2009
@@ -18,23 +18,30 @@
 package org.apache.mahout.cf.taste.hadoop;
 
 import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.Reducer;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 
 public final class SlopeOnePrefsToDiffsReducer
-    extends Reducer<Text, ItemPrefWritable, ItemItemWritable, FloatWritable> {
+    extends MapReduceBase
+    implements Reducer<LongWritable, ItemPrefWritable, ItemItemWritable, FloatWritable>
{
 
   @Override
-  protected void reduce(Text key, Iterable<ItemPrefWritable> values, Context context)
-      throws IOException, InterruptedException {
+  public void reduce(LongWritable key,
+                     Iterator<ItemPrefWritable> values,
+                     OutputCollector<ItemItemWritable, FloatWritable> output,
+                     Reporter reporter) throws IOException {
     List<ItemPrefWritable> prefs = new ArrayList<ItemPrefWritable>();
-    for (ItemPrefWritable value : values) {
-      prefs.add(new ItemPrefWritable(value));
+    while (values.hasNext()) {
+      prefs.add(new ItemPrefWritable(values.next()));
     }
     Collections.sort(prefs, ByItemIDComparator.getInstance());
     int size = prefs.size();
@@ -46,7 +53,7 @@
         ItemPrefWritable second = prefs.get(j);
         long itemBID = second.getItemID();
         float itemBValue = second.getPrefValue();
-        context.write(new ItemItemWritable(itemAID, itemBID), new FloatWritable(itemBValue
- itemAValue));
+        output.collect(new ItemItemWritable(itemAID, itemBID), new FloatWritable(itemBValue
- itemAValue));
       }
     }
   }

Modified: lucene/mahout/trunk/taste-web/pom.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/taste-web/pom.xml?rev=885782&r1=885781&r2=885782&view=diff
==============================================================================
--- lucene/mahout/trunk/taste-web/pom.xml (original)
+++ lucene/mahout/trunk/taste-web/pom.xml Tue Dec  1 15:02:25 2009
@@ -123,7 +123,7 @@
           <artifactId>hbase</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>org.apache.mahout.hadoop</groupId>
+          <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-core</artifactId>
         </exclusion>
         <exclusion>



Mime
View raw message