mahout-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sro...@apache.org
Subject svn commit: r803081 [2/7] - in /lucene/mahout/trunk: core/src/main/java/org/apache/mahout/cf/taste/eval/ core/src/main/java/org/apache/mahout/cf/taste/hadoop/ core/src/main/java/org/apache/mahout/cf/taste/impl/common/ core/src/main/java/org/apache/maho...
Date Tue, 11 Aug 2009 12:04:38 GMT
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java (from r802574, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingIterator.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingIterator.java&r1=802574&r2=803081&rev=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingIterator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java Tue Aug 11 12:04:35 2009
@@ -25,16 +25,16 @@
  * Wraps an {@link Iterator} and returns only some subset of the elements that it would, as determined by a sampling
  * rate parameter.
  */
-public final class SamplingIterator<T> implements Iterator<T> {
+public final class SamplingLongPrimitiveIterator implements LongPrimitiveIterator {
 
   private static final Random r = RandomUtils.getRandom();
 
-  private final Iterator<? extends T> delegate;
+  private final LongPrimitiveIterator delegate;
   private final double samplingRate;
-  private T next;
+  private long next;
   private boolean hasNext;
 
-  public SamplingIterator(Iterator<? extends T> delegate, double samplingRate) {
+  public SamplingLongPrimitiveIterator(LongPrimitiveIterator delegate, double samplingRate) {
     this.delegate = delegate;
     this.samplingRate = samplingRate;
     this.hasNext = true;
@@ -47,19 +47,32 @@
   }
 
   @Override
-  public T next() {
+  public Long next() {
+    return nextLong();
+  }
+
+  @Override
+  public long nextLong() {
     if (hasNext) {
-      T result = next;
+      long result = next;
       doNext();
       return result;
     }
     throw new NoSuchElementException();
   }
 
+  @Override
+  public long peek() {
+    if (hasNext) {
+      return next;
+    }
+    throw new NoSuchElementException();
+  }
+
   private void doNext() {
     boolean found = false;
     if (delegate instanceof SkippingIterator) {
-      SkippingIterator<? extends T> skippingDelegate = (SkippingIterator<? extends T>) delegate;
+      SkippingIterator<?> skippingDelegate = (SkippingIterator<?>) delegate;
       int toSkip = 0;
       while (r.nextDouble() >= samplingRate) {
         toSkip++;
@@ -69,12 +82,12 @@
         skippingDelegate.skip(toSkip);
       }
       if (skippingDelegate.hasNext()) {
-        next = skippingDelegate.next();
+        next = delegate.next();
         found = true;
       }
     } else {
       while (delegate.hasNext()) {
-        T delegateNext = delegate.next();
+        long delegateNext = delegate.next();
         if (r.nextDouble() < samplingRate) {
           next = delegateNext;
           found = true;
@@ -84,7 +97,6 @@
     }
     if (!found) {
       hasNext = false;
-      next = null;
     }
   }
 
@@ -96,4 +108,8 @@
     throw new UnsupportedOperationException();
   }
 
-}
+  public static LongPrimitiveIterator maybeWrapIterator(LongPrimitiveIterator delegate, double samplingRate) {
+    return samplingRate >= 1.0 ? delegate : new SamplingLongPrimitiveIterator(delegate, samplingRate);
+  }
+
+}
\ No newline at end of file

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java Tue Aug 11 12:04:35 2009
@@ -18,12 +18,15 @@
 package org.apache.mahout.cf.taste.impl.eval;
 
 import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.eval.DataModelBuilder;
 import org.apache.mahout.cf.taste.eval.RecommenderBuilder;
 import org.apache.mahout.cf.taste.eval.RecommenderEvaluator;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
 import org.apache.mahout.cf.taste.impl.common.RandomUtils;
 import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
 import org.apache.mahout.cf.taste.impl.model.GenericPreference;
+import org.apache.mahout.cf.taste.impl.model.GenericUserPreferenceArray;
 import org.apache.mahout.cf.taste.model.DataModel;
 import org.apache.mahout.cf.taste.model.Preference;
 import org.apache.mahout.cf.taste.model.PreferenceArray;
@@ -32,9 +35,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
 
 /** <p>Abstract superclass of a couple implementations, providing shared functionality.</p> */
@@ -50,6 +51,7 @@
 
   @Override
   public double evaluate(RecommenderBuilder recommenderBuilder,
+                         DataModelBuilder dataModelBuilder,
                          DataModel dataModel,
                          double trainingPercentage,
                          double evaluationPercentage) throws TasteException {
@@ -70,18 +72,21 @@
     log.info("Beginning evaluation using " + trainingPercentage + " of " + dataModel);
 
     int numUsers = dataModel.getNumUsers();
-    Map<Comparable<?>, Collection<Preference>> trainingUsers =
-            new FastMap<Comparable<?>, Collection<Preference>>(1 + (int) (trainingPercentage * (double) numUsers));
-    Map<Comparable<?>, Collection<Preference>> testUserPrefs =
-            new FastMap<Comparable<?>, Collection<Preference>>(1 + (int) ((1.0 - trainingPercentage) * (double) numUsers));
+    FastByIDMap<PreferenceArray> trainingUsers =
+            new FastByIDMap<PreferenceArray>(1 + (int) (trainingPercentage * (double) numUsers));
+    FastByIDMap<PreferenceArray> testUserPrefs =
+            new FastByIDMap<PreferenceArray>(1 + (int) ((1.0 - trainingPercentage) * (double) numUsers));
 
-    for (Comparable<?> userID : dataModel.getUserIDs()) {
+    LongPrimitiveIterator it = dataModel.getUserIDs();
+    while (it.hasNext()) {
       if (random.nextDouble() < evaluationPercentage) {
-        processOneUser(trainingPercentage, trainingUsers, testUserPrefs, userID, dataModel);
+        processOneUser(trainingPercentage, trainingUsers, testUserPrefs, it.nextLong(), dataModel);
       }
     }
 
-    DataModel trainingModel = new GenericDataModel(GenericDataModel.toPrefArrayValues(trainingUsers, true));
+    DataModel trainingModel = dataModelBuilder == null ?
+            new GenericDataModel(trainingUsers) :
+            dataModelBuilder.buildDataModel(trainingUsers);
 
     Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
 
@@ -91,33 +96,37 @@
   }
 
   private void processOneUser(double trainingPercentage,
-                              Map<Comparable<?>, Collection<Preference>> trainingUsers,
-                              Map<Comparable<?>, Collection<Preference>> testUserPrefs,
-                              Comparable<?> userID,
+                              FastByIDMap<PreferenceArray> trainingUsers,
+                              FastByIDMap<PreferenceArray> testUserPrefs,
+                              long userID,
                               DataModel dataModel) throws TasteException {
-    List<Preference> trainingPrefs = new ArrayList<Preference>();
-    List<Preference> testPrefs = new ArrayList<Preference>();
+    List<Preference> trainingPrefs = null;
+    List<Preference> testPrefs = null;
     PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
     int size = prefs.length();
     for (int i = 0; i < size; i++) {
       Preference newPref = new GenericPreference(userID, prefs.getItemID(i), prefs.getValue(i));
       if (random.nextDouble() < trainingPercentage) {
+        if (trainingPrefs == null) {
+          trainingPrefs = new ArrayList<Preference>(3);
+        }
         trainingPrefs.add(newPref);
       } else {
+        if (testPrefs == null) {
+          testPrefs = new ArrayList<Preference>(3);
+        }
         testPrefs.add(newPref);
       }
     }
-    log.debug("Training against {} preferences", trainingPrefs.size());
-    log.debug("Evaluating accuracy of {} preferences", testPrefs.size());
-    if (!trainingPrefs.isEmpty()) {
-      trainingUsers.put(userID, trainingPrefs);
-      if (!testPrefs.isEmpty()) {
-        testUserPrefs.put(userID, testPrefs);
+    if (trainingPrefs != null) {
+      trainingUsers.put(userID, new GenericUserPreferenceArray(trainingPrefs));
+      if (testPrefs != null) {
+        testUserPrefs.put(userID, new GenericUserPreferenceArray(testPrefs));
       }
     }
   }
 
-  abstract double getEvaluation(Map<Comparable<?>, Collection<Preference>> testUserPrefs, Recommender recommender)
+  abstract double getEvaluation(FastByIDMap<PreferenceArray> testUserPrefs, Recommender recommender)
       throws TasteException;
 
 }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java Tue Aug 11 12:04:35 2009
@@ -20,14 +20,15 @@
 import org.apache.mahout.cf.taste.common.NoSuchItemException;
 import org.apache.mahout.cf.taste.common.NoSuchUserException;
 import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
 import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
 import org.apache.mahout.cf.taste.impl.common.RunningAverage;
 import org.apache.mahout.cf.taste.model.Preference;
+import org.apache.mahout.cf.taste.model.PreferenceArray;
 import org.apache.mahout.cf.taste.recommender.Recommender;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
 import java.util.Map;
 
 /**
@@ -41,13 +42,11 @@
   private static final Logger log = LoggerFactory.getLogger(AverageAbsoluteDifferenceRecommenderEvaluator.class);
 
   @Override
-  double getEvaluation(Map<Comparable<?>, Collection<Preference>> testUserPrefs,
-                       Recommender recommender)
-      throws TasteException {
+  double getEvaluation(FastByIDMap<PreferenceArray> testUserPrefs, Recommender recommender) throws TasteException {
     RunningAverage average = new FullRunningAverage();
-    for (Map.Entry<Comparable<?>, Collection<Preference>> entry : testUserPrefs.entrySet()) {
+    for (Map.Entry<Long, PreferenceArray> entry : testUserPrefs.entrySet()) {
       for (Preference realPref : entry.getValue()) {
-        Comparable<?> testUserID = entry.getKey();
+        long testUserID = entry.getKey();
         try {
           float estimatedPreference =
               recommender.estimatePreference(testUserID, realPref.getItemID());
@@ -57,9 +56,9 @@
         } catch (NoSuchUserException nsue) {
           // It's possible that an item exists in the test data but not training data in which case
           // NSEE will be thrown. Just ignore it and move on.
-          log.debug("User exists in test data but not training data: {}", testUserID, nsue);
+          log.info("User exists in test data but not training data: {}", testUserID);
         } catch (NoSuchItemException nsie) {
-          log.debug("Item exists in test data but not training data: {}", realPref.getItemID(), nsie);
+          log.info("Item exists in test data but not training data: {}", realPref.getItemID());
         }
       }
     }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java Tue Aug 11 12:04:35 2009
@@ -19,17 +19,20 @@
 
 import org.apache.mahout.cf.taste.common.NoSuchUserException;
 import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.eval.DataModelBuilder;
 import org.apache.mahout.cf.taste.eval.IRStatistics;
 import org.apache.mahout.cf.taste.eval.RecommenderBuilder;
 import org.apache.mahout.cf.taste.eval.RecommenderIRStatsEvaluator;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
 import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
 import org.apache.mahout.cf.taste.impl.common.FullRunningAverageAndStdDev;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
 import org.apache.mahout.cf.taste.impl.common.RandomUtils;
 import org.apache.mahout.cf.taste.impl.common.RunningAverage;
 import org.apache.mahout.cf.taste.impl.common.RunningAverageAndStdDev;
 import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
+import org.apache.mahout.cf.taste.impl.model.GenericUserPreferenceArray;
 import org.apache.mahout.cf.taste.model.DataModel;
 import org.apache.mahout.cf.taste.model.Preference;
 import org.apache.mahout.cf.taste.model.PreferenceArray;
@@ -40,10 +43,8 @@
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Random;
 
 /**
@@ -58,8 +59,9 @@
   private static final Logger log = LoggerFactory.getLogger(GenericRecommenderIRStatsEvaluator.class);
 
   /**
-   * Pass as "relevanceThreshold" argument to {@link #evaluate(RecommenderBuilder, DataModel, Rescorer, int, double,
-   * double)} to have it attempt to compute a reasonable threshold. Note that this will impact performance.
+   * Pass as "relevanceThreshold" argument to
+   * {@link #evaluate(RecommenderBuilder, DataModelBuilder, DataModel, Rescorer, int, double, double)}
+   * to have it attempt to compute a reasonable threshold. Note that this will impact performance.
    */
   public static final double CHOOSE_THRESHOLD = Double.NaN;
 
@@ -71,8 +73,9 @@
 
   @Override
   public IRStatistics evaluate(RecommenderBuilder recommenderBuilder,
+                               DataModelBuilder dataModelBuilder,
                                DataModel dataModel,
-                               Rescorer<Comparable<?>> rescorer,
+                               Rescorer<Long> rescorer,
                                int at,
                                double relevanceThreshold,
                                double evaluationPercentage) throws TasteException {
@@ -97,10 +100,12 @@
     RunningAverage precision = new FullRunningAverage();
     RunningAverage recall = new FullRunningAverage();
     RunningAverage fallOut = new FullRunningAverage();
-    for (Comparable<?> userID : dataModel.getUserIDs()) {
+    LongPrimitiveIterator it = dataModel.getUserIDs();
+    while (it.hasNext()) {
+      long userID = it.nextLong();
       if (random.nextDouble() < evaluationPercentage) {
         long start = System.currentTimeMillis();
-        Collection<Comparable<?>> relevantItemIDs = new FastSet<Comparable<?>>(at);
+        FastIDSet relevantItemIDs = new FastIDSet(at);
         PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
         int size = prefs.length();
         if (size < 2 * at) {
@@ -118,13 +123,16 @@
         }
         int numRelevantItems = relevantItemIDs.size();
         if (numRelevantItems > 0) {
-          Map<Comparable<?>, Collection<Preference>> trainingUsers =
-            new FastMap<Comparable<?>, Collection<Preference>>(dataModel.getNumUsers());
-          for (Comparable userID2 : dataModel.getUserIDs()) {
-            processOtherUser(userID, relevantItemIDs, trainingUsers, userID2, dataModel);
+          FastByIDMap<PreferenceArray> trainingUsers =
+            new FastByIDMap<PreferenceArray>(dataModel.getNumUsers());
+          LongPrimitiveIterator it2 = dataModel.getUserIDs();
+          while (it2.hasNext()) {
+            processOtherUser(userID, relevantItemIDs, trainingUsers, it2.nextLong(), dataModel);
           }
 
-          DataModel trainingModel = new GenericDataModel(GenericDataModel.toPrefArrayValues(trainingUsers, true));
+          DataModel trainingModel = dataModelBuilder == null ?
+            new GenericDataModel(trainingUsers) :
+            dataModelBuilder.buildDataModel(trainingUsers);
           Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
 
           try {
@@ -162,25 +170,26 @@
     return new IRStatisticsImpl(precision.getAverage(), recall.getAverage(), fallOut.getAverage());
   }
 
-  private static void processOtherUser(Comparable<?> id,
-                                       Collection<Comparable<?>> relevantItemIDs,
-                                       Map<Comparable<?>, Collection<Preference>> trainingUsers,
-                                       Comparable<?> userID2,
+  private static void processOtherUser(long id,
+                                       FastIDSet relevantItemIDs,
+                                       FastByIDMap<PreferenceArray> trainingUsers,
+                                       long userID2,
                                        DataModel dataModel) throws TasteException {
     PreferenceArray prefs2Array = dataModel.getPreferencesFromUser(userID2);
-    List<Preference> prefs2 = new ArrayList<Preference>(prefs2Array.length());
-    for (Preference pref : prefs2Array) {
-      prefs2.add(pref);
-    }
-    if (id.equals(userID2)) {
+    if (id == userID2) {
+      List<Preference> prefs2 = new ArrayList<Preference>(prefs2Array.length());
+      for (Preference pref : prefs2Array) {
+        prefs2.add(pref);
+      }
       for (Iterator<Preference> iterator = prefs2.iterator(); iterator.hasNext();) {
         Preference pref = iterator.next();
         if (relevantItemIDs.contains(pref.getItemID())) {
           iterator.remove();
         }
       }
+      prefs2Array = new GenericUserPreferenceArray(prefs2);
     }
-    trainingUsers.put(userID2, prefs2);
+    trainingUsers.put(userID2, prefs2Array);
   }
 
   private static double computeThreshold(PreferenceArray prefs) {

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java Tue Aug 11 12:04:35 2009
@@ -35,11 +35,10 @@
     DataModel dataModel = recommender.getDataModel();
     int numUsers = dataModel.getNumUsers();
     double sampleRate = 1000.0 / numUsers;
-    SamplingIterable<Comparable<?>> userSampler =
-            new SamplingIterable<Comparable<?>>(dataModel.getUserIDs(), sampleRate);
+    SamplingIterable<Long> userSampler = new SamplingIterable<Long>(dataModel.getUserIDs(), sampleRate);
     RunningAverageAndStdDev recommendationTime = new FullRunningAverageAndStdDev();
     int count = 0;
-    for (Comparable<?> userID : userSampler) {
+    for (long userID : userSampler) {
       long start = System.currentTimeMillis();
       recommender.recommend(userID, 10);
       long end = System.currentTimeMillis();

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java Tue Aug 11 12:04:35 2009
@@ -19,14 +19,15 @@
 
 import org.apache.mahout.cf.taste.common.NoSuchUserException;
 import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
 import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
 import org.apache.mahout.cf.taste.impl.common.RunningAverage;
 import org.apache.mahout.cf.taste.model.Preference;
+import org.apache.mahout.cf.taste.model.PreferenceArray;
 import org.apache.mahout.cf.taste.recommender.Recommender;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Collection;
 import java.util.Map;
 
 /**
@@ -39,13 +40,11 @@
   private static final Logger log = LoggerFactory.getLogger(RMSRecommenderEvaluator.class);
 
   @Override
-  double getEvaluation(Map<Comparable<?>, Collection<Preference>> testUserPrefs,
-                       Recommender recommender)
-      throws TasteException {
+  double getEvaluation(FastByIDMap<PreferenceArray> testUserPrefs, Recommender recommender) throws TasteException {
     RunningAverage average = new FullRunningAverage();
-    for (Map.Entry<Comparable<?>, Collection<Preference>> entry : testUserPrefs.entrySet()) {
+    for (Map.Entry<Long, PreferenceArray> entry : testUserPrefs.entrySet()) {
       for (Preference realPref : entry.getValue()) {
-        Comparable<?> testUserID = entry.getKey();
+        long testUserID = entry.getKey();
         try {
           float estimatedPreference =
               recommender.estimatePreference(testUserID, realPref.getItemID());
@@ -56,7 +55,7 @@
         } catch (NoSuchUserException nsee) {
           // It's possible that an item exists in the test data but not training data in which case
           // NSEE will be thrown. Just ignore it and move on.
-          log.info("Element exists in test data but not training data: {}", testUserID, nsee);
+          log.info("User exists in test data but not training data: {}", testUserID);
         }
       }
     }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -21,6 +21,7 @@
 import org.apache.mahout.cf.taste.model.PreferenceArray;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
@@ -34,14 +35,14 @@
  */
 public final class BooleanItemPreferenceArray implements PreferenceArray, Serializable {
 
-  private final Comparable<?>[] IDs;
-  private Comparable<?> id;
+  private final long[] IDs;
+  private long id;
 
   public BooleanItemPreferenceArray(int size) {
     if (size < 1) {
       throw new IllegalArgumentException("size is less than 1");
     }
-    this.IDs = new Comparable<?>[size];
+    this.IDs = new long[size];
   }
 
   public int length() {
@@ -69,22 +70,22 @@
   }
 
   @Override
-  public Comparable<?> getUserID(int i) {
+  public long getUserID(int i) {
     return IDs[i];
   }
 
   @Override
-  public void setUserID(int i, Comparable<?> userID) {
+  public void setUserID(int i, long userID) {
     IDs[i] = userID;
   }
 
   @Override
-  public Comparable<?> getItemID(int i) {
+  public long getItemID(int i) {
     return id;
   }
 
   @Override
-  public void setItemID(int i, Comparable<?> itemID) {
+  public void setItemID(int i, long itemID) {
     id = itemID;
   }
 
@@ -100,7 +101,7 @@
 
   @Override
   public void sortByUser() {
-    selectionSort();
+    Arrays.sort(IDs);
   }
 
   @Override
@@ -115,24 +116,6 @@
   public void sortByValueReversed() {
   }
 
-  private void selectionSort() {
-    // I think this sort will prove to be too dumb, but, it's in place and OK for tiny, mostly sorted data
-    int max = length();
-    for (int i = 0; i < max; i++) {
-      int min = i;
-      for (int j = i + 1; j < max; j++) {
-        if (((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0) {
-          min = j;
-        }
-      }
-      if (i != min) {
-        Comparable<?> temp1 = IDs[i];
-        IDs[i] = IDs[min];
-        IDs[min] = temp1;
-      }
-    }
-  }
-
   public GenericItemPreferenceArray clone() {
     try {
       return (GenericItemPreferenceArray) super.clone();
@@ -171,12 +154,12 @@
     }
 
     @Override
-    public Comparable<?> getUserID() {
+    public long getUserID() {
       return BooleanItemPreferenceArray.this.getUserID(i);
     }
 
     @Override
-    public Comparable<?> getItemID() {
+    public long getItemID() {
       return BooleanItemPreferenceArray.this.getItemID(i);
     }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java Tue Aug 11 12:04:35 2009
@@ -28,24 +28,21 @@
  */
 public final class BooleanPreference implements Preference, Serializable {
 
-  private final Comparable<?> userID;
-  private final Comparable<?> itemID;
+  private final long userID;
+  private final long itemID;
 
-  public BooleanPreference(Comparable<?> userID, Comparable<?> itemID) {
-    if (userID == null || itemID == null) {
-      throw new IllegalArgumentException("itemID is null");
-    }
+  public BooleanPreference(long userID, long itemID) {
     this.userID = userID;
     this.itemID = itemID;
   }
 
   @Override
-  public Comparable<?> getUserID() {
+  public long getUserID() {
     return userID;
   }
 
   @Override
-  public Comparable<?> getItemID() {
+  public long getItemID() {
     return itemID;
   }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -21,6 +21,7 @@
 import org.apache.mahout.cf.taste.model.PreferenceArray;
 
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
@@ -34,14 +35,14 @@
  */
 public final class BooleanUserPreferenceArray implements PreferenceArray, Serializable {
 
-  private final Comparable<?>[] IDs;
-  private Comparable<?> id;
+  private final long[] IDs;
+  private long id;
 
   public BooleanUserPreferenceArray(int size) {
     if (size < 1) {
       throw new IllegalArgumentException("size is less than 1");
     }
-    this.IDs = new Comparable<?>[size];
+    this.IDs = new long[size];
   }
 
   public int length() {
@@ -69,22 +70,22 @@
   }
 
   @Override
-  public Comparable<?> getUserID(int i) {
+  public long getUserID(int i) {
     return id;
   }
 
   @Override
-  public void setUserID(int i, Comparable<?> userID) {
+  public void setUserID(int i, long userID) {
     id = userID;
   }
 
   @Override
-  public Comparable<?> getItemID(int i) {
+  public long getItemID(int i) {
     return IDs[i];
   }
 
   @Override
-  public void setItemID(int i, Comparable<?> itemID) {
+  public void setItemID(int i, long itemID) {
     IDs[i] = itemID;
   }
 
@@ -104,7 +105,7 @@
 
   @Override
   public void sortByItem() {
-    selectionSort();
+    Arrays.sort(IDs);
   }
 
   @Override
@@ -115,24 +116,6 @@
   public void sortByValueReversed() {
   }
 
-  private void selectionSort() {
-    // I think this sort will prove to be too dumb, but, it's in place and OK for tiny, mostly sorted data
-    int max = length();
-    for (int i = 0; i < max; i++) {
-      int min = i;
-      for (int j = i + 1; j < max; j++) {
-        if (((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0) {
-          min = j;
-        }
-      }
-      if (i != min) {
-        Comparable<?> temp1 = IDs[i];
-        IDs[i] = IDs[min];
-        IDs[min] = temp1;
-      }
-    }
-  }
-
   public GenericItemPreferenceArray clone() {
     try {
       return (GenericItemPreferenceArray) super.clone();
@@ -171,12 +154,12 @@
     }
 
     @Override
-    public Comparable<?> getUserID() {
+    public long getUserID() {
       return BooleanUserPreferenceArray.this.getUserID(i);
     }
 
     @Override
-    public Comparable<?> getItemID() {
+    public long getItemID() {
       return BooleanUserPreferenceArray.this.getItemID(i);
     }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java Tue Aug 11 12:04:35 2009
@@ -21,15 +21,16 @@
 import org.apache.mahout.cf.taste.common.NoSuchUserException;
 import org.apache.mahout.cf.taste.common.Refreshable;
 import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveArrayIterator;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
 import org.apache.mahout.cf.taste.model.DataModel;
 import org.apache.mahout.cf.taste.model.PreferenceArray;
 
 import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -40,10 +41,10 @@
  */
 public final class GenericBooleanPrefDataModel implements DataModel, Serializable {
 
-  private final List<Comparable<?>> userIDs;
-  private final Map<Comparable<?>, FastSet<Comparable<?>>> preferenceFromUsers;
-  private final List<Comparable<?>> itemIDs;
-  private final Map<Comparable<?>, FastSet<Comparable<?>>> preferenceForItems;
+  private final long[] userIDs;
+  private final FastByIDMap<FastIDSet> preferenceFromUsers;
+  private final long[] itemIDs;
+  private final FastByIDMap<FastIDSet> preferenceForItems;
 
   /**
    * <p>Creates a new {@link GenericDataModel} from the given users (and their preferences). This {@link
@@ -52,34 +53,41 @@
    * @param userData users to include
    */
   @SuppressWarnings("unchecked")
-  public GenericBooleanPrefDataModel(Map<Comparable<?>, FastSet<Comparable<?>>> userData) {
+  public GenericBooleanPrefDataModel(FastByIDMap<FastIDSet> userData) {
     if (userData == null) {
       throw new IllegalArgumentException("userData is null");
     }
 
     this.preferenceFromUsers = userData;
-    this.preferenceForItems = new FastMap<Comparable<?>, FastSet<Comparable<?>>>();
-    FastSet<Comparable<?>> itemIDSet = new FastSet<Comparable<?>>();
-    for (Map.Entry<Comparable<?>, FastSet<Comparable<?>>> entry : preferenceFromUsers.entrySet()) {
-      Comparable<?> userID = entry.getKey();
-      FastSet<Comparable<?>> itemIDs = entry.getValue();
+    this.preferenceForItems = new FastByIDMap<FastIDSet>();
+    FastIDSet itemIDSet = new FastIDSet();
+    for (Map.Entry<Long, FastIDSet> entry : preferenceFromUsers.entrySet()) {
+      long userID = entry.getKey();
+      FastIDSet itemIDs = entry.getValue();
       itemIDSet.addAll(itemIDs);
-      for (Comparable<?> itemID : itemIDs) {
-        FastSet<Comparable<?>> userIDs = preferenceForItems.get(itemID);
+      LongPrimitiveIterator it = itemIDs.iterator();
+      while (it.hasNext()) {
+        long itemID = it.next();
+        FastIDSet userIDs = preferenceForItems.get(itemID);
         if (userIDs == null) {
-          userIDs = new FastSet<Comparable<?>>(2);
+          userIDs = new FastIDSet(2);
           preferenceForItems.put(itemID, userIDs);
         }
         userIDs.add(userID);
       }
     }
 
-    this.itemIDs = new ArrayList<Comparable<?>>(itemIDSet);
-    itemIDSet = null;
-    Collections.sort((List<? extends Comparable>) this.itemIDs);
+    this.itemIDs = itemIDSet.toArray();
+    itemIDSet = null; // Might help GC -- this is big
+    Arrays.sort(itemIDs);
 
-    this.userIDs = new ArrayList(userData.keySet());
-    Collections.sort((List<? extends Comparable>) userIDs);
+    this.userIDs = new long[userData.size()];
+    int i = 0;
+    LongPrimitiveIterator it = userData.keySetIterator();
+    while (it.hasNext()) {
+      userIDs[i++] = it.next();
+    }
+    Arrays.sort(userIDs);
 
   }
 
@@ -94,40 +102,55 @@
     this(toDataMap(dataModel));
   }
 
-  private static Map<Comparable<?>, FastSet<Comparable<?>>> toDataMap(DataModel dataModel) throws TasteException {
-    Map<Comparable<?>, FastSet<Comparable<?>>> data = 
-            new FastMap<Comparable<?>, FastSet<Comparable<?>>>(dataModel.getNumUsers());
-    for (Comparable<?> userID : dataModel.getUserIDs()) {
+  private static FastByIDMap<FastIDSet> toDataMap(DataModel dataModel) throws TasteException {
+    FastByIDMap<FastIDSet> data = new FastByIDMap<FastIDSet>(dataModel.getNumUsers());
+    LongPrimitiveIterator it = dataModel.getUserIDs();
+    while (it.hasNext()) {
+      long userID = it.nextLong();
       data.put(userID, dataModel.getItemIDsFromUser(userID));
     }
     return data;
   }
 
+  public static FastByIDMap<FastIDSet> toDataMap(FastByIDMap<PreferenceArray> data) throws TasteException {
+    for (Map.Entry<Long, Object> entry : ((FastByIDMap<Object>) (FastByIDMap<?>) data).entrySet()) {
+      PreferenceArray prefArray = (PreferenceArray) entry.getValue();
+      int size = prefArray.length();
+      FastIDSet itemIDs = new FastIDSet(size);
+      for (int i = 0; i < size; i++) {
+        itemIDs.add(prefArray.getItemID(i));
+      }
+      entry.setValue(itemIDs);
+    }
+    return (FastByIDMap<FastIDSet>) (FastByIDMap<?>) data;
+  }
+
   @Override
-  public Iterable<Comparable<?>> getUserIDs() {
-    return userIDs;
+  public LongPrimitiveArrayIterator getUserIDs() {
+    return new LongPrimitiveArrayIterator(userIDs);
   }
 
   /** @throws NoSuchUserException if there is no such user */
   @Override
-  public PreferenceArray getPreferencesFromUser(Comparable<?> userID) throws NoSuchUserException {
-    FastSet<Comparable<?>> itemIDs = preferenceFromUsers.get(userID);
+  public PreferenceArray getPreferencesFromUser(long userID) throws NoSuchUserException {
+    FastIDSet itemIDs = preferenceFromUsers.get(userID);
     if (itemIDs == null) {
       throw new NoSuchUserException();
     }
     PreferenceArray prefArray = new BooleanUserPreferenceArray(itemIDs.size());
     int i = 0;
-    for (Comparable<?> itemID : itemIDs) {
+    LongPrimitiveIterator it = itemIDs.iterator();
+    while (it.hasNext()) {
       prefArray.setUserID(i, userID);
-      prefArray.setItemID(i, itemID);
+      prefArray.setItemID(i, it.next());
       i++;
     }
     return prefArray;
   }
 
   @Override
-  public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> userID) throws TasteException {
-    FastSet<Comparable<?>> itemIDs = preferenceFromUsers.get(userID);
+  public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
+    FastIDSet itemIDs = preferenceFromUsers.get(userID);
     if (itemIDs == null) {
       throw new NoSuchUserException();
     }
@@ -135,20 +158,21 @@
   }
 
   @Override
-  public Iterable<Comparable<?>> getItemIDs() {
-    return itemIDs;
+  public LongPrimitiveArrayIterator getItemIDs() {
+    return new LongPrimitiveArrayIterator(itemIDs);
   }
 
   @Override
-  public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws NoSuchItemException {
-    FastSet<Comparable<?>> userIDs = preferenceForItems.get(itemID);
+  public PreferenceArray getPreferencesForItem(long itemID) throws NoSuchItemException {
+    FastIDSet userIDs = preferenceForItems.get(itemID);
     if (userIDs == null) {
       throw new NoSuchItemException();
     }
     PreferenceArray prefArray = new BooleanItemPreferenceArray(userIDs.size());
     int i = 0;
-    for (Comparable<?> userID : userIDs) {
-      prefArray.setUserID(i, userID);
+    LongPrimitiveIterator it = userIDs.iterator();
+    while (it.hasNext()) {
+      prefArray.setUserID(i, it.next());
       prefArray.setItemID(i, itemID);
       i++;
     }
@@ -156,8 +180,8 @@
   }
 
   @Override
-  public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws NoSuchUserException, NoSuchItemException {
-    FastSet<Comparable<?>> itemIDs = preferenceFromUsers.get(userID);
+  public Float getPreferenceValue(long userID, long itemID) throws NoSuchUserException, NoSuchItemException {
+    FastIDSet itemIDs = preferenceFromUsers.get(userID);
     if (itemIDs == null) {
       throw new NoSuchUserException();
     }
@@ -169,24 +193,24 @@
 
   @Override
   public int getNumItems() {
-    return itemIDs.size();
+    return itemIDs.length;
   }
 
   @Override
   public int getNumUsers() {
-    return userIDs.size();
+    return userIDs.length;
   }
 
   @Override
-  public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) throws NoSuchItemException {
+  public int getNumUsersWithPreferenceFor(long... itemIDs) throws NoSuchItemException {
     if (itemIDs.length == 0) {
       return 0;
     }
-    FastSet<Comparable<?>> intersection = new FastSet<Comparable<?>>();
-    FastSet<Comparable<?>> userIDs = preferenceForItems.get(itemIDs[0]);
+    FastIDSet userIDs = preferenceForItems.get(itemIDs[0]);
     if (userIDs == null) {
       throw new NoSuchItemException();
     }
+    FastIDSet intersection = new FastIDSet(userIDs.size());
     intersection.addAll(userIDs);
     int i = 1;
     while (!intersection.isEmpty() && i < itemIDs.length) {
@@ -200,12 +224,12 @@
   }
 
   @Override
-  public void removePreference(Comparable<?> userID, Comparable<?> itemID) throws NoSuchUserException {
+  public void removePreference(long userID, long itemID) throws NoSuchUserException {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value) throws NoSuchUserException {
+  public void setPreference(long userID, long itemID, float value) throws NoSuchUserException {
     throw new UnsupportedOperationException();
   }
 
@@ -216,7 +240,19 @@
 
   @Override
   public String toString() {
-    return "GenericBooleanPrefDataModel[users:" + (userIDs.size() > 3 ? userIDs.subList(0, 3) + "..." : userIDs) + ']';
+    StringBuilder result = new StringBuilder(200);
+    result.append("GenericDataModel[users:");
+    for (int i = 0; i < Math.min(3, userIDs.length); i++) {
+      if (i > 0) {
+        result.append(',');
+      }
+      result.append(userIDs[i]);
+    }
+    if (result.length() > 3) {
+      result.append("...");
+    }
+    result.append(']');
+    return result.toString();
   }
 
 }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java Tue Aug 11 12:04:35 2009
@@ -21,8 +21,10 @@
 import org.apache.mahout.cf.taste.common.NoSuchUserException;
 import org.apache.mahout.cf.taste.common.Refreshable;
 import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveArrayIterator;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
 import org.apache.mahout.cf.taste.model.DataModel;
 import org.apache.mahout.cf.taste.model.Preference;
 import org.apache.mahout.cf.taste.model.PreferenceArray;
@@ -31,11 +33,10 @@
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 /**
  * <p>A simple {@link DataModel} which uses a given {@link List} of users as its data source. This
@@ -46,33 +47,33 @@
 
   private static final Logger log = LoggerFactory.getLogger(GenericDataModel.class);
 
-  private final List<Comparable<?>> userIDs;
-  private final Map<Comparable<?>, PreferenceArray> preferenceFromUsers;
-  private final List<Comparable<?>> itemIDs;
-  private final Map<Comparable<?>, PreferenceArray> preferenceForItems;
+  private final long[] userIDs;
+  private final FastByIDMap<PreferenceArray> preferenceFromUsers;
+  private final long[] itemIDs;
+  private final FastByIDMap<PreferenceArray> preferenceForItems;
 
   /**
    * <p>Creates a new {@link GenericDataModel} from the given users (and their preferences). This {@link
    * DataModel} retains all this information in memory and is effectively immutable.</p>
    *
    * @param userData users to include in this {@link GenericDataModel}
-   *  (see also {@link #toPrefArrayValues(Map, boolean)})
+   *  (see also {@link #toDataMap(FastByIDMap, boolean)})
    */
   @SuppressWarnings("unchecked")
-  public GenericDataModel(Map<Comparable<?>, PreferenceArray> userData) {
+  public GenericDataModel(FastByIDMap<PreferenceArray> userData) {
     if (userData == null) {
       throw new IllegalArgumentException("userData is null");
     }
 
     this.preferenceFromUsers = userData;
-    FastMap<Comparable<?>, Collection<Preference>> prefsForItems = new FastMap<Comparable<?>, Collection<Preference>>();
-    FastSet<Comparable<?>> itemIDSet = new FastSet<Comparable<?>>();
+    FastByIDMap<Collection<Preference>> prefsForItems = new FastByIDMap<Collection<Preference>>();
+    FastIDSet itemIDSet = new FastIDSet();
     int currentCount = 0;
-    for (PreferenceArray prefs : preferenceFromUsers.values()) {
+    for (Map.Entry<Long, PreferenceArray> entry : preferenceFromUsers.entrySet()) {
+      PreferenceArray prefs = entry.getValue();
       prefs.sortByItem();
-      int size = prefs.length();
       for (Preference preference : prefs) {
-        Comparable<?> itemID = preference.getItemID();
+        long itemID = preference.getItemID();
         itemIDSet.add(itemID);
         List<Preference> prefsForItem = (List<Preference>) prefsForItems.get(itemID);
         if (prefsForItem == null) {
@@ -81,24 +82,28 @@
         }
         prefsForItem.add(preference);
       }
-      currentCount++;
-      if (currentCount % 10000 == 0) {
+      if (++currentCount % 10000 == 0) {
         log.info("Processed {} users", currentCount);
       }
     }
 
-    this.itemIDs = new ArrayList<Comparable<?>>(itemIDSet);
-    itemIDSet = null;
-    Collections.sort((List<? extends Comparable>) this.itemIDs);
-
-    this.preferenceForItems = toPrefArrayValues(prefsForItems, false);
-
-    for (PreferenceArray prefs : preferenceForItems.values()) {
-      prefs.sortByUser();
-    }
-    
-    this.userIDs = new ArrayList(userData.keySet());
-    Collections.sort((List<? extends Comparable>) userIDs);
+    this.itemIDs = itemIDSet.toArray();
+    itemIDSet = null; // Might help GC -- this is big
+    Arrays.sort(itemIDs);
+
+    this.preferenceForItems = toDataMap(prefsForItems, false);
+
+    for (Map.Entry<Long, PreferenceArray> entry : preferenceForItems.entrySet()) {
+      entry.getValue().sortByUser();
+    }
+
+    this.userIDs = new long[userData.size()];
+    int i = 0;
+    LongPrimitiveIterator it = userData.keySetIterator();
+    while (it.hasNext()) {
+      userIDs[i++] = it.next();
+    }
+    Arrays.sort(userIDs);
   }
 
   /**
@@ -117,32 +122,32 @@
    * .
    * @return input value
    */
-  public static Map<Comparable<?>, PreferenceArray> toPrefArrayValues(Map<Comparable<?>, Collection<Preference>> data,
-                                                                      boolean byUser) {
-    for (Map.Entry<Comparable<?>, Object> entry :
-         ((Map<Comparable<?>, Object>) (Map<Comparable<?>, ?>) data).entrySet()) {
+  public static FastByIDMap<PreferenceArray> toDataMap(FastByIDMap<Collection<Preference>> data, boolean byUser) {
+    for (Map.Entry<Long, Object> entry : ((FastByIDMap<Object>) (FastByIDMap<?>) data).entrySet()) {
       List<Preference> prefList = (List<Preference>) entry.getValue();
       entry.setValue(byUser ? new GenericUserPreferenceArray(prefList) : new GenericItemPreferenceArray(prefList));
     }
-    return (Map<Comparable<?>, PreferenceArray>) (Map<Comparable<?>, ?>) data;
+    return (FastByIDMap<PreferenceArray>) (FastByIDMap<?>) data;
   }
 
-  private static Map<Comparable<?>, PreferenceArray> toDataMap(DataModel dataModel) throws TasteException {
-    Map<Comparable<?>, PreferenceArray> data = new FastMap<Comparable<?>, PreferenceArray>(dataModel.getNumUsers());
-    for (Comparable<?> userID : dataModel.getUserIDs()) {
+  private static FastByIDMap<PreferenceArray> toDataMap(DataModel dataModel) throws TasteException {
+    FastByIDMap<PreferenceArray> data = new FastByIDMap<PreferenceArray>(dataModel.getNumUsers());
+    LongPrimitiveIterator it = dataModel.getUserIDs();
+    while (it.hasNext()) {
+      long userID = it.nextLong();
       data.put(userID, dataModel.getPreferencesFromUser(userID));
     }
     return data;
   }
 
   @Override
-  public Iterable<Comparable<?>> getUserIDs() {
-    return userIDs;
+  public LongPrimitiveArrayIterator getUserIDs() {
+    return new LongPrimitiveArrayIterator(userIDs);
   }
 
   /** @throws NoSuchUserException if there is no such user */
   @Override
-  public PreferenceArray getPreferencesFromUser(Comparable<?> userID) throws NoSuchUserException {
+  public PreferenceArray getPreferencesFromUser(long userID) throws NoSuchUserException {
     PreferenceArray prefs = preferenceFromUsers.get(userID);
     if (prefs == null) {
       throw new NoSuchUserException();
@@ -151,10 +156,10 @@
   }
 
   @Override
-  public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> userID) throws TasteException {
+  public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
     PreferenceArray prefs = getPreferencesFromUser(userID);
     int size = prefs.length();
-    FastSet<Comparable<?>> result = new FastSet<Comparable<?>>(size);
+    FastIDSet result = new FastIDSet(size);
     for (int i = 0; i < size; i++) {
       result.add(prefs.getItemID(i));
     }
@@ -162,12 +167,12 @@
   }
 
   @Override
-  public Iterable<Comparable<?>> getItemIDs() {
-    return itemIDs;
+  public LongPrimitiveArrayIterator getItemIDs() {
+    return new LongPrimitiveArrayIterator(itemIDs);
   }
 
   @Override
-  public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws NoSuchItemException {
+  public PreferenceArray getPreferencesForItem(long itemID) throws NoSuchItemException {
     PreferenceArray prefs = preferenceForItems.get(itemID);
     if (prefs == null) {
       throw new NoSuchItemException();
@@ -176,11 +181,11 @@
   }
 
   @Override
-  public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+  public Float getPreferenceValue(long userID, long itemID) throws TasteException {
     PreferenceArray prefs = getPreferencesFromUser(userID);
     int size = prefs.length();
     for (int i = 0; i < size; i++) {
-      if (prefs.getItemID(i).equals(itemID)) {
+      if (prefs.getItemID(i) == itemID) {
         return prefs.getValue(i);
       }
     }
@@ -189,16 +194,16 @@
 
   @Override
   public int getNumItems() {
-    return itemIDs.size();
+    return itemIDs.length;
   }
 
   @Override
   public int getNumUsers() {
-    return userIDs.size();
+    return userIDs.length;
   }
 
   @Override
-  public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) {
+  public int getNumUsersWithPreferenceFor(long... itemIDs) {
     if (itemIDs == null) {
       throw new IllegalArgumentException("itemIDs is null");
     }
@@ -215,12 +220,12 @@
       if (prefs1 == null || prefs2 == null) {
         return 0;
       }
-      Set<Comparable<?>> users1 = new FastSet<Comparable<?>>(prefs1.length());
+      FastIDSet users1 = new FastIDSet(prefs1.length());
       int size1 = prefs1.length();
       for (int i = 0; i < size1; i++) {
         users1.add(prefs1.getUserID(i));
       }
-      Set<Comparable<?>> users2 = new FastSet<Comparable<?>>(prefs2.length());
+      FastIDSet users2 = new FastIDSet(prefs2.length());
       int size2 = prefs2.length();
       for (int i = 0; i < size2; i++) {
         users2.add(prefs2.getUserID(i));
@@ -231,12 +236,12 @@
   }
 
   @Override
-  public void removePreference(Comparable<?> userID, Comparable<?> itemID) throws NoSuchUserException {
+  public void removePreference(long userID, long itemID) throws NoSuchUserException {
     throw new UnsupportedOperationException();
   }
 
   @Override
-  public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value) throws NoSuchUserException {
+  public void setPreference(long userID, long itemID, float value) throws NoSuchUserException {
     throw new UnsupportedOperationException();
   }
 
@@ -247,7 +252,19 @@
 
   @Override
   public String toString() {
-    return "GenericDataModel[users:" + (userIDs.size() > 3 ? userIDs.subList(0, 3) + "..." : userIDs) + ']';
+    StringBuilder result = new StringBuilder(200);
+    result.append("GenericDataModel[users:");
+    for (int i = 0; i < Math.min(3, userIDs.length); i++) {
+      if (i > 0) {
+        result.append(',');
+      }
+      result.append(userIDs[i]);
+    }
+    if (result.length() > 3) {
+      result.append("...");
+    }
+    result.append(']');
+    return result.toString();
   }
 
 }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -38,15 +38,15 @@
   private static final int VALUE = 2;
   private static final int VALUE_REVERSED = 3;
 
-  private final Comparable<?>[] IDs;
-  private Comparable<?> id;
+  private final long[] IDs;
+  private long id;
   private final float[] values;
 
   public GenericItemPreferenceArray(int size) {
     if (size < 1) {
       throw new IllegalArgumentException("size is less than 1");
     }
-    this.IDs = new Comparable<?>[size];
+    this.IDs = new long[size];
     values = new float[size];
   }
 
@@ -77,22 +77,22 @@
   }
 
   @Override
-  public Comparable<?> getUserID(int i) {
+  public long getUserID(int i) {
     return IDs[i];
   }
 
   @Override
-  public void setUserID(int i, Comparable<?> userID) {
+  public void setUserID(int i, long userID) {
     IDs[i] = userID;
   }
 
   @Override
-  public Comparable<?> getItemID(int i) {
+  public long getItemID(int i) {
     return id;
   }
 
   @Override
-  public void setItemID(int i, Comparable<?> itemID) {
+  public void setItemID(int i, long itemID) {
     id = itemID;
   }
 
@@ -144,7 +144,7 @@
   private boolean isLess(int i, int j, int type) {
     switch (type) {
       case USER:
-        return ((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0;
+        return IDs[i] < IDs[j];
       case VALUE:
         return values[i] < values[j];
       case VALUE_REVERSED:
@@ -155,7 +155,7 @@
   }
 
   private void swap(int i, int j) {
-    Comparable<?> temp1 = IDs[i];
+    long temp1 = IDs[i];
     float temp2 = values[i];
     IDs[i] = IDs[j];
     values[i] = values[j];
@@ -201,12 +201,12 @@
     }
 
     @Override
-    public Comparable<?> getUserID() {
+    public long getUserID() {
       return GenericItemPreferenceArray.this.getUserID(i);
     }
 
     @Override
-    public Comparable<?> getItemID() {
+    public long getItemID() {
       return GenericItemPreferenceArray.this.getItemID(i);
     }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java Tue Aug 11 12:04:35 2009
@@ -24,14 +24,11 @@
 /** <p>A simple {@link Preference} encapsulating an item and preference value.</p> */
 public class GenericPreference implements Preference, Serializable {
 
-  private final Comparable<?> userID;
-  private final Comparable<?> itemID;
+  private final long userID;
+  private final long itemID;
   private float value;
 
-  public GenericPreference(Comparable<?> userID, Comparable<?> itemID, float value) {
-    if (userID == null || itemID == null) {
-      throw new IllegalArgumentException("userID or itemID is null");
-    }
+  public GenericPreference(long userID, long itemID, float value) {
     if (Float.isNaN(value)) {
       throw new IllegalArgumentException("Invalid value: " + value);
     }
@@ -41,12 +38,12 @@
   }
 
   @Override
-  public Comparable<?> getUserID() {
+  public long getUserID() {
     return userID;
   }
 
   @Override
-  public Comparable<?> getItemID() {
+  public long getItemID() {
     return itemID;
   }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -39,15 +39,15 @@
   private static final int VALUE = 2;
   private static final int VALUE_REVERSED = 3;
 
-  private final Comparable<?>[] IDs;
-  private Comparable<?> id;
+  private final long[] IDs;
+  private long id;
   private final float[] values;
 
   public GenericUserPreferenceArray(int size) {
     if (size < 1) {
       throw new IllegalArgumentException("size is less than 1");
     }
-    this.IDs = new Comparable<?>[size];
+    this.IDs = new long[size];
     values = new float[size];
   }
 
@@ -78,22 +78,22 @@
   }
 
   @Override
-  public Comparable<?> getUserID(int i) {
+  public long getUserID(int i) {
     return id;
   }
 
   @Override
-  public void setUserID(int i, Comparable<?> userID) {
+  public void setUserID(int i, long userID) {
     id = userID;
   }
 
   @Override
-  public Comparable<?> getItemID(int i) {
+  public long getItemID(int i) {
     return IDs[i];
   }
 
   @Override
-  public void setItemID(int i, Comparable<?> itemID) {
+  public void setItemID(int i, long itemID) {
     IDs[i] = itemID;
   }
 
@@ -145,7 +145,7 @@
   private boolean isLess(int i, int j, int type) {
     switch (type) {
       case ITEM:
-        return ((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0;
+        return IDs[i] < IDs[j];
       case VALUE:
         return values[i] < values[j];
       case VALUE_REVERSED:
@@ -156,7 +156,7 @@
   }
 
   private void swap(int i, int j) {
-    Comparable<?> temp1 = IDs[i];
+    long temp1 = IDs[i];
     float temp2 = values[i];
     IDs[i] = IDs[j];
     values[i] = values[j];
@@ -202,12 +202,12 @@
     }
 
     @Override
-    public Comparable<?> getUserID() {
+    public long getUserID() {
       return GenericUserPreferenceArray.this.getUserID(i);
     }
 
     @Override
-    public Comparable<?> getItemID() {
+    public long getItemID() {
       return GenericUserPreferenceArray.this.getItemID(i);
     }
 

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java Tue Aug 11 12:04:35 2009
@@ -19,9 +19,10 @@
 
 import org.apache.mahout.cf.taste.common.Refreshable;
 import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
 import org.apache.mahout.cf.taste.impl.common.FileLineIterator;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
 import org.apache.mahout.cf.taste.impl.model.GenericBooleanPrefDataModel;
 import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
 import org.apache.mahout.cf.taste.impl.model.GenericPreference;
@@ -75,8 +76,8 @@
  * that, a JDBC-backed {@link DataModel} and a database are more appropriate.</p>
  *
  * <p>It is possible and likely useful to subclass this class and customize its behavior to accommodate
- * application-specific needs and input formats. See {@link #processLine(String, Map, char)} and
- * {@link #processLineWithoutID(String, Map, char)}
+ * application-specific needs and input formats. See {@link #processLine(String, FastByIDMap, char)} and
+ * {@link #processLineWithoutID(String, FastByIDMap, char)}
  */
 public class FileDataModel implements DataModel {
 
@@ -138,17 +139,17 @@
     FileLineIterator iterator = new FileLineIterator(dataFile, false);
     String firstLine = iterator.peek();
     char delimiter = determineDelimiter(firstLine);
-    boolean hasPrefValues = firstLine.indexOf(',', firstLine.indexOf(',') + 1) >= 0;
+    boolean hasPrefValues = firstLine.indexOf(delimiter, firstLine.indexOf(delimiter) + 1) >= 0;
 
     if (hasPrefValues) {
-      Map<Comparable<?>, Collection<Preference>> data = new FastMap<Comparable<?>, Collection<Preference>>();
+      FastByIDMap<Collection<Preference>> data = new FastByIDMap<Collection<Preference>>();
       processFile(iterator, data, delimiter);
       for (File updateFile : findUpdateFiles()) {
         processFile(new FileLineIterator(updateFile, false), data, delimiter);
       }
-      return new GenericDataModel(GenericDataModel.toPrefArrayValues(data, true));
+      return new GenericDataModel(GenericDataModel.toDataMap(data, true));
     } else {
-      Map<Comparable<?>, FastSet<Comparable<?>>> data = new FastMap<Comparable<?>, FastSet<Comparable<?>>>();
+      FastByIDMap<FastIDSet> data = new FastByIDMap<FastIDSet>();
       processFileWithoutID(iterator, data, delimiter);
       for (File updateFile : findUpdateFiles()) {
         processFileWithoutID(new FileLineIterator(updateFile, false), data, delimiter);
@@ -189,7 +190,7 @@
   }
 
   protected void processFile(FileLineIterator dataOrUpdateFileIterator,
-                             Map<Comparable<?>, Collection<Preference>> data,
+                             FastByIDMap<Collection<Preference>> data,
                              char delimiter) {
     log.info("Reading file info...");
     AtomicInteger count = new AtomicInteger();
@@ -217,7 +218,7 @@
    * @param line      line from input data file
    * @param data      all data read so far, as a mapping from user IDs to preferences
    */
-  protected void processLine(String line, Map<Comparable<?>, Collection<Preference>> data, char delimiter) {
+  protected void processLine(String line, FastByIDMap<Collection<Preference>> data, char delimiter) {
 
     if (line.length() == 0 || line.charAt(0) == '#') {
       return;
@@ -229,12 +230,12 @@
       throw new IllegalArgumentException("Bad line: " + line);
     }
 
-    String userID = line.substring(0, delimiterOne);
-    String itemID = line.substring(delimiterOne + 1, delimiterTwo);
+    long userID = Long.parseLong(line.substring(0, delimiterOne));
+    long itemID = Long.parseLong(line.substring(delimiterOne + 1, delimiterTwo));
     String preferenceValueString = line.substring(delimiterTwo + 1);
 
     if (transpose) {
-      String tmp = userID;
+      long tmp = userID;
       userID = itemID;
       itemID = tmp;
     }
@@ -249,7 +250,7 @@
       Iterator<Preference> prefsIterator = prefs.iterator();
       while (prefsIterator.hasNext()) {
         Preference pref = prefsIterator.next();
-        if (pref.getItemID().equals(itemID)) {
+        if (pref.getItemID() == itemID) {
           prefsIterator.remove();
           break;
         }
@@ -261,7 +262,7 @@
   }
 
   protected void processFileWithoutID(FileLineIterator dataOrUpdateFileIterator,
-                                      Map<Comparable<?>, FastSet<Comparable<?>>> data,
+                                      FastByIDMap<FastIDSet> data,
                                       char delimiter) {
     log.info("Reading file info...");
     AtomicInteger count = new AtomicInteger();
@@ -278,7 +279,7 @@
     log.info("Read lines: {}", count.get());
   }
 
-  protected void processLineWithoutID(String line, Map<Comparable<?>, FastSet<Comparable<?>>> data, char delimiter) {
+  protected void processLineWithoutID(String line, FastByIDMap<FastIDSet> data, char delimiter) {
 
     if (line.length() == 0 || line.charAt(0) == '#') {
       return;
@@ -289,17 +290,17 @@
       throw new IllegalArgumentException("Bad line: " + line);
     }
 
-    String userID = line.substring(0, delimiterOne);
-    String itemID = line.substring(delimiterOne + 1);
+    long userID = Long.parseLong(line.substring(0, delimiterOne));
+    long itemID = Long.parseLong(line.substring(delimiterOne + 1));
 
     if (transpose) {
-      String tmp = userID;
+      long tmp = userID;
       userID = itemID;
       itemID = tmp;
     }
-    FastSet<Comparable<?>> itemIDs = data.get(userID);
+    FastIDSet itemIDs = data.get(userID);
     if (itemIDs == null) {
-      itemIDs = new FastSet<Comparable<?>>(2);
+      itemIDs = new FastIDSet(2);
       data.put(userID, itemIDs);
     }
     itemIDs.add(itemID);
@@ -312,36 +313,36 @@
   }
 
   @Override
-  public Iterable<Comparable<?>> getUserIDs() throws TasteException {
+  public LongPrimitiveIterator getUserIDs() throws TasteException {
     checkLoaded();
     return delegate.getUserIDs();
   }
 
   @Override
-  public PreferenceArray getPreferencesFromUser(Comparable<?> userID) throws TasteException {
+  public PreferenceArray getPreferencesFromUser(long userID) throws TasteException {
     checkLoaded();
     return delegate.getPreferencesFromUser(userID);
   }
 
   @Override
-  public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> userID) throws TasteException {
+  public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
     return delegate.getItemIDsFromUser(userID);
   }
 
   @Override
-  public Iterable<Comparable<?>> getItemIDs() throws TasteException {
+  public LongPrimitiveIterator getItemIDs() throws TasteException {
     checkLoaded();
     return delegate.getItemIDs();
   }
 
   @Override
-  public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws TasteException {
+  public PreferenceArray getPreferencesForItem(long itemID) throws TasteException {
     checkLoaded();
     return delegate.getPreferencesForItem(itemID);
   }
 
   @Override
-  public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+  public Float getPreferenceValue(long userID, long itemID) throws TasteException {
     return delegate.getPreferenceValue(userID, itemID);
   }
 
@@ -358,7 +359,7 @@
   }
 
   @Override
-  public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) throws TasteException {
+  public int getNumUsersWithPreferenceFor(long... itemIDs) throws TasteException {
     checkLoaded();
     return delegate.getNumUsersWithPreferenceFor(itemIDs);
   }
@@ -369,14 +370,14 @@
    * reloaded from a file. This method should also be considered relatively slow.
    */
   @Override
-  public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value) throws TasteException {
+  public void setPreference(long userID, long itemID, float value) throws TasteException {
     checkLoaded();
     delegate.setPreference(userID, itemID, value);
   }
 
-  /** See the warning at {@link #setPreference(Comparable, Comparable, float)}. */
+  /** See the warning at {@link #setPreference(long, long, float)}. */
   @Override
-  public void removePreference(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+  public void removePreference(long userID, long itemID) throws TasteException {
     checkLoaded();
     delegate.removePreference(userID, itemID);
   }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -75,15 +75,11 @@
 
   @Override
   protected Preference buildPreference(ResultSet rs) throws SQLException {
-    return new BooleanPreference((Comparable<?>) rs.getObject(1), (Comparable<?>) rs.getObject(2));
+    return new BooleanPreference(rs.getLong(1), rs.getLong(2));
   }
 
   @Override
-  public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value)
-      throws TasteException {
-    if (userID == null || itemID == null) {
-      throw new IllegalArgumentException("userID or itemID is null");
-    }
+  public void setPreference(long userID, long itemID, float value) throws TasteException {
     if (!Float.isNaN(value)) {
       throw new IllegalArgumentException("Invalid value: " + value);
     }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -21,10 +21,10 @@
 import org.apache.mahout.cf.taste.common.Refreshable;
 import org.apache.mahout.cf.taste.common.TasteException;
 import org.apache.mahout.cf.taste.impl.common.Cache;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
 import org.apache.mahout.cf.taste.impl.common.IOUtils;
-import org.apache.mahout.cf.taste.impl.common.IteratorIterable;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
 import org.apache.mahout.cf.taste.impl.common.Retriever;
 import org.apache.mahout.cf.taste.impl.common.SkippingIterator;
 import org.apache.mahout.cf.taste.impl.common.jdbc.AbstractJDBCComponent;
@@ -47,7 +47,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.NoSuchElementException;
 
 /**
@@ -91,7 +90,7 @@
   private final String getNumPreferenceForItemsSQL;
   private int cachedNumUsers;
   private int cachedNumItems;
-  private final Cache<Comparable<?>, Integer> itemPrefCounts;
+  private final Cache<Long, Integer> itemPrefCounts;
 
   protected AbstractJDBCDataModel(DataSource dataSource,
                                   String getPreferenceSQL,
@@ -190,7 +189,7 @@
 
     this.cachedNumUsers = -1;
     this.cachedNumItems = -1;
-    this.itemPrefCounts = new Cache<Comparable<?>, Integer>(new ItemPrefCountRetriever(getNumPreferenceForItemSQL));
+    this.itemPrefCounts = new Cache<Long, Integer>(new ItemPrefCountRetriever(getNumPreferenceForItemSQL));
 
   }
 
@@ -217,14 +216,14 @@
   }
 
   @Override
-  public Iterable<Comparable<?>> getUserIDs() throws TasteException {
+  public LongPrimitiveIterator getUserIDs() throws TasteException {
     log.debug("Retrieving all users...");
-    return new IteratorIterable<Comparable<?>>(new ResultSetIDIterator(getUsersSQL));
+    return new ResultSetIDIterator(getUsersSQL);
   }
 
   /** @throws NoSuchUserException if there is no such user */
   @Override
-  public PreferenceArray getPreferencesFromUser(Comparable<?> id) throws TasteException {
+  public PreferenceArray getPreferencesFromUser(long id) throws TasteException {
 
     log.debug("Retrieving user ID '{}'", id);
 
@@ -237,7 +236,7 @@
       stmt = conn.prepareStatement(getUserSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
       stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
       stmt.setFetchSize(getFetchSize());
-      stmt.setObject(1, id);
+      stmt.setLong(1, id);
 
       log.debug("Executing SQL query: {}", getUserSQL);
       rs = stmt.executeQuery();
@@ -263,14 +262,14 @@
   }
 
   @Override
-  public Map<Comparable<?>, PreferenceArray> exportWithPrefs() throws TasteException {
+  public FastByIDMap<PreferenceArray> exportWithPrefs() throws TasteException {
     log.debug("Exporting all data");
 
     Connection conn = null;
     Statement stmt = null;
     ResultSet rs = null;
 
-    Map<Comparable<?>, PreferenceArray> result = new FastMap<Comparable<?>, PreferenceArray>();
+    FastByIDMap<PreferenceArray> result = new FastByIDMap<PreferenceArray>();
 
     try {
       conn = dataSource.getConnection();
@@ -281,10 +280,10 @@
       log.debug("Executing SQL query: {}", getAllUsersSQL);
       rs = stmt.executeQuery(getAllUsersSQL);
 
-      Comparable<?> currentUserID = null;
+      Long currentUserID = null;
       List<Preference> currentPrefs = new ArrayList<Preference>();
       while (rs.next()) {
-        Comparable<?> nextUserID = (Comparable<?>) rs.getObject(1);
+        long nextUserID = rs.getLong(1);
         if (currentUserID != null && !currentUserID.equals(nextUserID)) {
           if (!currentPrefs.isEmpty()) {
             result.put(currentUserID, new GenericUserPreferenceArray(currentPrefs));
@@ -311,14 +310,14 @@
   }
 
   @Override
-  public Map<Comparable<?>, FastSet<Comparable<?>>> exportWithIDsOnly() throws TasteException {
+  public FastByIDMap<FastIDSet> exportWithIDsOnly() throws TasteException {
     log.debug("Exporting all data");
 
     Connection conn = null;
     Statement stmt = null;
     ResultSet rs = null;
 
-    Map<Comparable<?>, FastSet<Comparable<?>>> result = new FastMap<Comparable<?>, FastSet<Comparable<?>>>();
+    FastByIDMap<FastIDSet> result = new FastByIDMap<FastIDSet>();
 
     try {
       conn = dataSource.getConnection();
@@ -329,19 +328,21 @@
       log.debug("Executing SQL query: {}", getAllUsersSQL);
       rs = stmt.executeQuery(getAllUsersSQL);
 
-      Comparable<?> currentUserID = null;
-      FastSet<Comparable<?>> currentItemIDs = new FastSet<Comparable<?>>(2);
+      boolean currentUserIDSet = false;
+      long currentUserID = 0L; // value isn't used
+      FastIDSet currentItemIDs = new FastIDSet(2);
       while (rs.next()) {
-        Comparable<?> nextUserID = (Comparable<?>) rs.getObject(1);
-        if (currentUserID != null && !currentUserID.equals(nextUserID)) {
+        long nextUserID = rs.getLong(1);
+        if (currentUserIDSet && currentUserID != nextUserID) {
           if (!currentItemIDs.isEmpty()) {
             result.put(currentUserID, currentItemIDs);
-            currentItemIDs = new FastSet<Comparable<?>>(2);
+            currentItemIDs = new FastIDSet(2);
           }
         } else {
-          currentItemIDs.add((Comparable<?>) rs.getObject(2));
+          currentItemIDs.add(rs.getLong(2));
         }
         currentUserID = nextUserID;
+        currentUserIDSet = true;
       }
       if (!currentItemIDs.isEmpty()) {
         result.put(currentUserID, currentItemIDs);
@@ -360,7 +361,7 @@
 
   /** @throws NoSuchUserException if there is no such user */
   @Override
-  public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> id) throws TasteException {
+  public FastIDSet getItemIDsFromUser(long id) throws TasteException {
 
     log.debug("Retrieving items for user ID '{}'", id);
 
@@ -373,14 +374,14 @@
       stmt = conn.prepareStatement(getUserSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
       stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
       stmt.setFetchSize(getFetchSize());
-      stmt.setObject(1, id);
+      stmt.setLong(1, id);
 
       log.debug("Executing SQL query: {}", getUserSQL);
       rs = stmt.executeQuery();
 
-      FastSet<Comparable<?>> result = new FastSet<Comparable<?>>();
+      FastIDSet result = new FastIDSet();
       while (rs.next()) {
-        result.add((Comparable<?>) rs.getObject(1));
+        result.add(rs.getLong(1));
       }
 
       if (result.isEmpty()) {
@@ -399,7 +400,7 @@
   }
 
   @Override
-  public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+  public Float getPreferenceValue(long userID, long itemID) throws TasteException {
     log.debug("Retrieving preferences for item ID '{}'", itemID);
     Connection conn = null;
     PreparedStatement stmt = null;
@@ -409,8 +410,8 @@
       stmt = conn.prepareStatement(getPreferenceSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
       stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
       stmt.setFetchSize(1);
-      stmt.setObject(1, userID);
-      stmt.setObject(2, itemID);
+      stmt.setLong(1, userID);
+      stmt.setLong(2, itemID);
 
       log.debug("Executing SQL query: {}", getPreferenceSQL);
       rs = stmt.executeQuery();
@@ -428,18 +429,18 @@
   }
 
   @Override
-  public Iterable<Comparable<?>> getItemIDs() throws TasteException {
+  public LongPrimitiveIterator getItemIDs() throws TasteException {
     log.debug("Retrieving all items...");
-    return new IteratorIterable<Comparable<?>>(new ResultSetIDIterator(getItemsSQL));
+    return new ResultSetIDIterator(getItemsSQL);
   }
 
   @Override
-  public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws TasteException {
+  public PreferenceArray getPreferencesForItem(long itemID) throws TasteException {
     List<Preference> list = doGetPreferencesForItem(itemID);
     return new GenericItemPreferenceArray(list);
   }
 
-  protected List<Preference> doGetPreferencesForItem(Comparable<?> itemID) throws TasteException {
+  protected List<Preference> doGetPreferencesForItem(long itemID) throws TasteException {
     log.debug("Retrieving preferences for item ID '{}'", itemID);
     Connection conn = null;
     PreparedStatement stmt = null;
@@ -449,7 +450,7 @@
       stmt = conn.prepareStatement(getPrefsForItemSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
       stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
       stmt.setFetchSize(getFetchSize());
-      stmt.setObject(1, itemID);
+      stmt.setLong(1, itemID);
 
       log.debug("Executing SQL query: {}", getPrefsForItemSQL);
       rs = stmt.executeQuery();
@@ -483,7 +484,7 @@
   }
 
   @Override
-  public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) throws TasteException {
+  public int getNumUsersWithPreferenceFor(long... itemIDs) throws TasteException {
     if (itemIDs == null) {
       throw new IllegalArgumentException("itemIDs is null");
     }
@@ -493,11 +494,11 @@
     }
     return length == 1 ?
         itemPrefCounts.get(itemIDs[0]) :
-        getNumThings("user preferring items", getNumPreferenceForItemsSQL, (Object[]) itemIDs);
+        getNumThings("user preferring items", getNumPreferenceForItemsSQL, itemIDs);
   }
 
 
-  private int getNumThings(String name, String sql, Object... args) throws TasteException {
+  private int getNumThings(String name, String sql, long... args) throws TasteException {
     log.debug("Retrieving number of {} in model", name);
     Connection conn = null;
     PreparedStatement stmt = null;
@@ -509,7 +510,7 @@
       stmt.setFetchSize(getFetchSize());
       if (args != null) {
         for (int i = 1; i <= args.length; i++) {
-          stmt.setObject(i, args[i - 1]);
+          stmt.setLong(i, args[i - 1]);
         }
       }
       log.debug("Executing SQL query: {}", sql);
@@ -525,11 +526,7 @@
   }
 
   @Override
-  public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value)
-      throws TasteException {
-    if (userID == null || itemID == null) {
-      throw new IllegalArgumentException("userID or itemID is null");
-    }
+  public void setPreference(long userID, long itemID, float value) throws TasteException {
     if (Float.isNaN(value)) {
       throw new IllegalArgumentException("Invalid value: " + value);
     }
@@ -542,8 +539,8 @@
     try {
       conn = dataSource.getConnection();
       stmt = conn.prepareStatement(setPreferenceSQL);
-      stmt.setObject(1, userID);
-      stmt.setObject(2, itemID);
+      stmt.setLong(1, userID);
+      stmt.setLong(2, itemID);
       stmt.setDouble(3, value);
       stmt.setDouble(4, value);
 
@@ -559,11 +556,7 @@
   }
 
   @Override
-  public void removePreference(Comparable<?> userID, Comparable<?> itemID)
-      throws TasteException {
-    if (userID == null || itemID == null) {
-      throw new IllegalArgumentException("userID or itemID is null");
-    }
+  public void removePreference(long userID, long itemID) throws TasteException {
 
     log.debug("Removing preference for user '{}', item '{}'", userID, itemID);
 
@@ -573,8 +566,8 @@
     try {
       conn = dataSource.getConnection();
       stmt = conn.prepareStatement(removePreferenceSQL);
-      stmt.setObject(1, userID);
-      stmt.setObject(2, itemID);
+      stmt.setLong(1, userID);
+      stmt.setLong(2, itemID);
 
       log.debug("Executing SQL update: {}", removePreferenceSQL);
       stmt.executeUpdate();
@@ -595,7 +588,7 @@
   }
 
   protected Preference buildPreference(ResultSet rs) throws SQLException {
-    return new GenericPreference((Comparable<?>) rs.getObject(1), (Comparable<?>) rs.getObject(2), rs.getFloat(3));
+    return new GenericPreference(rs.getLong(1), rs.getLong(2), rs.getFloat(3));
   }
 
   /**
@@ -605,7 +598,7 @@
    * only release database resources after {@link #hasNext()} has been called and has returned <code>false</code>;
    * callers should make sure to "drain" the entire set of data to avoid tying up database resources.</p>
    */
-  private final class ResultSetIDIterator implements SkippingIterator<Comparable<?>> {
+  private final class ResultSetIDIterator implements LongPrimitiveIterator, SkippingIterator<Long> {
 
     private final Connection connection;
     private final Statement statement;
@@ -648,15 +641,19 @@
       return nextExists;
     }
 
+    public Long next() {
+      return nextLong();
+    }
+
     @Override
-    public Comparable<?> next() {
+    public long nextLong() {
 
       if (!hasNext()) {
         throw new NoSuchElementException();
       }
 
       try {
-        Comparable<?> ID = (Comparable<?>) resultSet.getObject(1);
+        long ID = resultSet.getLong(1);
         resultSet.next();
         return ID;
       } catch (SQLException sqle) {
@@ -668,6 +665,22 @@
 
     }
 
+    @Override
+    public long peek() {
+      if (!hasNext()) {
+        throw new NoSuchElementException();
+      }
+      try {
+        return resultSet.getLong(1);
+      } catch (SQLException sqle) {
+        // No good way to handle this since we can't throw an exception
+        log.warn("Exception while iterating", sqle);
+        close();
+        throw new NoSuchElementException("Can't retrieve more due to exception: " + sqle);
+      }
+
+    }
+
     /**
      * @throws UnsupportedOperationException
      */
@@ -695,7 +708,7 @@
 
   }
 
-  private class ItemPrefCountRetriever implements Retriever<Comparable<?>, Integer> {
+  private class ItemPrefCountRetriever implements Retriever<Long, Integer> {
     private final String getNumPreferenceForItemSQL;
 
     private ItemPrefCountRetriever(String getNumPreferenceForItemSQL) {
@@ -703,7 +716,7 @@
     }
 
     @Override
-    public Integer get(Comparable<?> key) throws TasteException {
+    public Integer get(Long key) throws TasteException {
       return getNumThings("user preferring item", getNumPreferenceForItemSQL, key);
     }
   }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -28,8 +28,8 @@
  *
  * <pre>
  * CREATE TABLE taste_preferences (
- *   user_id INT NOT NULL,
- *   item_id INT NOT NULL,
+ *   user_id BIGINT NOT NULL,
+ *   item_id BIGINT NOT NULL,
  *   PRIMARY KEY (user_id, item_id),
  *   INDEX (user_id),
  *   INDEX (item_id)
@@ -122,7 +122,7 @@
         "SELECT COUNT(1) FROM " + preferenceTable + " WHERE " + itemIDColumn + "=?",
         // getNumPreferenceForItemsSQL
         "SELECT COUNT(1) FROM " + preferenceTable + " tp1 JOIN " + preferenceTable + " tp2 " +
-            "USING " + userIDColumn + " WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
+            "USING (" + userIDColumn + ") WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
   }
 
   @Override

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -31,21 +31,21 @@
  *
  * <table>
  * <tr><th>user_id</th><th>item_id</th><th>preference</th></tr>
- * <tr><td>ABC</td><td>123</td><td>0.9</td></tr>
- * <tr><td>ABC</td><td>456</td><td>0.1</td></tr>
- * <tr><td>DEF</td><td>123</td><td>0.2</td></tr>
- * <tr><td>DEF</td><td>789</td><td>0.3</td></tr>
+ * <tr><td>987</td><td>123</td><td>0.9</td></tr>
+ * <tr><td>987</td><td>456</td><td>0.1</td></tr>
+ * <tr><td>654</td><td>123</td><td>0.2</td></tr>
+ * <tr><td>654</td><td>789</td><td>0.3</td></tr>
  * </table>
  *
  * <p><code>preference</code> must have a type compatible
- * with the Java <code>float</code> type. <code>user_id</code> and <code>item_id</code> should be an integer or
- * string type (INT, LONGINT, VARCHAR). For example, the following command sets up a suitable table in MySQL,
+ * with the Java <code>float</code> type. <code>user_id</code> and <code>item_id</code> should be compatible
+ * with long type (BIGINT). For example, the following command sets up a suitable table in MySQL,
  * complete with primary key and indexes:</p>
  *
  * <pre>
  * CREATE TABLE taste_preferences (
- *   user_id INT NOT NULL,
- *   item_id INT NOT NULL,
+ *   user_id BIGINT NOT NULL,
+ *   item_id BIGINT NOT NULL,
  *   preference FLOAT NOT NULL,
  *   PRIMARY KEY (user_id, item_id),
  *   INDEX (user_id),
@@ -174,7 +174,7 @@
         "SELECT COUNT(1) FROM " + preferenceTable + " WHERE " + itemIDColumn + "=?",
         // getNumPreferenceForItemsSQL
         "SELECT COUNT(1) FROM " + preferenceTable + " tp1 JOIN " + preferenceTable + " tp2 " +
-            "USING " + userIDColumn + " WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
+            "USING (" + userIDColumn + ") WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
   }
 
   @Override



Mime
View raw message