mahout-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeast...@apache.org
Subject svn commit: r1006367 - in /mahout/trunk: core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/ core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/ core/src/main/java/org/apache/mahout/math/hadoop/decomposer/ examples/src/main...
Date Mon, 11 Oct 2010 01:03:06 GMT
Author: jeastman
Date: Mon Oct 11 01:03:06 2010
New Revision: 1006367

URL: http://svn.apache.org/viewvc?rev=1006367&view=rev
Log:
- Added DisplaySpectralKMeans example
- Made Configuration and JobConf handling explicit in EigenVerificationJob in order to make
the example work
- Updated SpectralKMeansDriver, EigencutsDriver to supply configs
- Updated example README
All tests run. DisplaySpectralKMeans example gets pretty far through before bombing on W.transpose()
after eigen cleanup. Committing for more eyeballs as is.

Added:
    mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java
Modified:
    mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java
    mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java
    mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
    mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/README.txt

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java?rev=1006367&r1=1006366&r2=1006367&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java
(original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java
Mon Oct 11 01:03:06 2010
@@ -129,7 +129,7 @@ public class EigencutsDriver extends Abs
       int overshoot = (int) ((double) dimensions * OVERSHOOT_MULTIPLIER);
       List<Double> eigenValues = new ArrayList<Double>(overshoot);
       Matrix eigenVectors = new DenseMatrix(overshoot, dimensions);
-      DistributedRowMatrix U = performEigenDecomposition(L, dimensions, overshoot, eigenValues,
eigenVectors, outputCalc);
+      DistributedRowMatrix U = performEigenDecomposition(conf, L, dimensions, overshoot,
eigenValues, eigenVectors, outputCalc);
       U.configure(new JobConf(conf));
       eigenValues = eigenValues.subList(0, dimensions);
 
@@ -164,15 +164,15 @@ public class EigencutsDriver extends Abs
    * values, and generally performing the tedious administrative tasks involved
    * in an eigen-decomposition and running the verifier
    */
-  public static DistributedRowMatrix performEigenDecomposition(DistributedRowMatrix input,
+  public static DistributedRowMatrix performEigenDecomposition(Configuration conf,
+                                                               DistributedRowMatrix input,
                                                                int numEigenVectors,
                                                                int overshoot,
                                                                List<Double> eigenValues,
-                                                               Matrix eigenVectors,
-                                                               Path tmp) throws IOException
{
+                                                               Matrix eigenVectors, Path
tmp) throws IOException {
     DistributedLanczosSolver solver = new DistributedLanczosSolver();
     Path seqFiles = new Path(tmp, "eigendecomp-" + (System.nanoTime() & 0xFF));
-    solver.runJob(new Configuration(),
+    solver.runJob(conf,
                   input.getRowPath(),
                   new Path(tmp, "lanczos-" + (System.nanoTime() & 0xFF)),
                   input.numRows(),
@@ -186,7 +186,7 @@ public class EigencutsDriver extends Abs
     // now run the verifier to trim down the number of eigenvectors
     EigenVerificationJob verifier = new EigenVerificationJob();
     Path verifiedEigens = new Path(tmp, "verifiedeigens");
-    verifier.runJob(seqFiles, input.getRowPath(), verifiedEigens, false, 1.0, 0.0, numEigenVectors);
+    verifier.runJob(conf, seqFiles, input.getRowPath(), verifiedEigens, false, 1.0, 0.0,
numEigenVectors);
     Path cleanedEigens = verifier.getCleanedEigensPath();
     return new DistributedRowMatrix(cleanedEigens, new Path(cleanedEigens, "tmp"), numEigenVectors,
input.numRows());
   }

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java?rev=1006367&r1=1006366&r2=1006367&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java
(original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java
Mon Oct 11 01:03:06 2010
@@ -146,7 +146,7 @@ public class SpectralKMeansDriver extend
     DistributedRowMatrix L =
         VectorMatrixMultiplicationJob.runJob(affSeqFiles, D,
             new Path(outputCalc, "laplacian-" + (System.nanoTime() & 0xFF)));
-    L.configure(new JobConf(conf));
+    L.configure(depConf);
 
     // Next step: perform eigen-decomposition using LanczosSolver
     // since some of the eigen-output is spurious and will be eliminated
@@ -171,10 +171,10 @@ public class SpectralKMeansDriver extend
     // perform a verification
     EigenVerificationJob verifier = new EigenVerificationJob();
     Path verifiedEigensPath = new Path(outputCalc, "eigenverifier");
-    verifier.runJob(lanczosSeqFiles, L.getRowPath(), verifiedEigensPath, true, 1.0, 0.0,
clusters);
+    verifier.runJob(conf, lanczosSeqFiles, L.getRowPath(), verifiedEigensPath, true, 1.0,
0.0, clusters);
     Path cleanedEigens = verifier.getCleanedEigensPath();
     DistributedRowMatrix W = new DistributedRowMatrix(cleanedEigens, new Path(cleanedEigens,
"tmp"), clusters, numDims);
-    W.configure(new JobConf());
+    W.configure(depConf);
     DistributedRowMatrix Wtrans = W.transpose();
     //    DistributedRowMatrix Wt = W.transpose();
 
@@ -182,7 +182,7 @@ public class SpectralKMeansDriver extend
     Path unitVectors = new Path(outputCalc, "unitvectors-" + (System.nanoTime() & 0xFF));
     UnitVectorizerJob.runJob(Wtrans.getRowPath(), unitVectors);
     DistributedRowMatrix Wt = new DistributedRowMatrix(unitVectors, new Path(unitVectors,
"tmp"), clusters, numDims);
-    Wt.configure(new JobConf());
+    Wt.configure(depConf);
 
     // Finally, perform k-means clustering on the rows of L (or W)
     // generate random initial clusters

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java?rev=1006367&r1=1006366&r2=1006367&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
(original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
Mon Oct 11 01:03:06 2010
@@ -88,8 +88,6 @@ public class EigenVerificationJob extend
 
   private Path outPath;
 
-  private JobConf conf;
-
   private int maxEigensToKeep;
 
   private Path cleanedEigensPath;
@@ -107,13 +105,13 @@ public class EigenVerificationJob extend
       return 0;
     }
     // parse out the arguments
-    runJob(new Path(argMap.get("--eigenInput")),
+    runJob(getConf(),
+           new Path(argMap.get("--eigenInput")),
            new Path(argMap.get("--corpusInput")),
            getOutputPath(),
            argMap.get("--inMemory") != null,
            Double.parseDouble(argMap.get("--maxError")),
-           Double.parseDouble(argMap.get("--minEigenvalue")),
-           Integer.parseInt(argMap.get("--maxEigens")));
+           Double.parseDouble(argMap.get("--minEigenvalue")), Integer.parseInt(argMap.get("--maxEigens")));
     return 0;
   }
 
@@ -141,13 +139,12 @@ public class EigenVerificationJob extend
     this.tmpOut = tempOut;
     this.maxError = maxError;
     this.minEigenValue = minEigenValue;
-    this.conf = config != null ? config : new JobConf();
 
     if (eigenInput != null && eigensToVerify == null) {
-      prepareEigens(eigenInput, inMemory);
+      prepareEigens(config, eigenInput, inMemory);
     }
     DistributedRowMatrix c = new DistributedRowMatrix(corpusInput, tempOut, 1, 1);
-    c.configure(conf);
+    c.configure(config);
     corpus = c;
 
     // set up eigenverifier and orthoverifier TODO: allow multithreaded execution
@@ -161,7 +158,7 @@ public class EigenVerificationJob extend
 
     List<Map.Entry<MatrixSlice, EigenStatus>> prunedEigenMeta = pruneEigens(eigenMetaData);
 
-    saveCleanEigens(prunedEigenMeta);
+    saveCleanEigens(new Configuration(), prunedEigenMeta);
     return 0;
   }
 
@@ -185,7 +182,7 @@ public class EigenVerificationJob extend
     return OrthonormalityVerifier.pairwiseInnerProducts(eigensToVerify);
   }
 
-  private void saveCleanEigens(List<Map.Entry<MatrixSlice, EigenStatus>> prunedEigenMeta)
throws IOException {
+  private void saveCleanEigens(Configuration conf, List<Map.Entry<MatrixSlice, EigenStatus>>
prunedEigenMeta) throws IOException {
     Path path = new Path(outPath, CLEAN_EIGENVECTORS);
     FileSystem fs = FileSystem.get(conf);
     SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class,
VectorWritable.class);
@@ -241,7 +238,7 @@ public class EigenVerificationJob extend
     return eigenMetaData;
   }
 
-  private void prepareEigens(Path eigenInput, boolean inMemory) {
+  private void prepareEigens(JobConf conf, Path eigenInput, boolean inMemory) {
     DistributedRowMatrix eigens = new DistributedRowMatrix(eigenInput, tmpOut, 1, 1);
     eigens.configure(conf);
     if (inMemory) {
@@ -267,6 +264,7 @@ public class EigenVerificationJob extend
 
   /**
    * Progammatic invocation of run()
+   * @param conf TODO
    * @param eigenInput Output of LanczosSolver
    * @param corpusInput Input of LanczosSolver
    * @param output
@@ -275,27 +273,24 @@ public class EigenVerificationJob extend
    * @param minEigenValue
    * @param maxEigens
    */
-  public void runJob(Path eigenInput,
+  public void runJob(Configuration conf,
+                     Path eigenInput,
                      Path corpusInput,
                      Path output,
                      boolean inMemory,
                      double maxError,
-                     double minEigenValue,
-                     int maxEigens) throws IOException {
+                     double minEigenValue, int maxEigens) throws IOException {
     // no need to handle command line arguments
     outPath = output;
     tmpOut = new Path(outPath, "tmp");
     maxEigensToKeep = maxEigens;
     this.maxError = maxError;
-    if (getConf() == null) {
-      setConf(new Configuration());
-    }
     if (eigenInput != null && eigensToVerify == null) {
-      prepareEigens(eigenInput, inMemory);
+      prepareEigens(new JobConf(conf), eigenInput, inMemory);
     }
 
     DistributedRowMatrix c = new DistributedRowMatrix(corpusInput, tmpOut, 1, 1);
-    c.configure(new JobConf(getConf()));
+    c.configure(new JobConf(conf));
     corpus = c;
 
     eigenVerifier = new SimpleEigenVerifier();
@@ -305,6 +300,6 @@ public class EigenVerificationJob extend
 
     Map<MatrixSlice, EigenStatus> eigenMetaData = verifyEigens();
     List<Map.Entry<MatrixSlice, EigenStatus>> prunedEigenMeta = pruneEigens(eigenMetaData);
-    saveCleanEigens(prunedEigenMeta);
+    saveCleanEigens(conf, prunedEigenMeta);
   }
 }

Added: mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java?rev=1006367&view=auto
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java
(added)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java
Mon Oct 11 01:03:06 2010
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.clustering.display;
+
+import java.awt.Graphics;
+import java.awt.Graphics2D;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.mahout.clustering.spectral.kmeans.SpectralKMeansDriver;
+import org.apache.mahout.common.HadoopUtil;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.distance.DistanceMeasure;
+import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
+
+class DisplaySpectralKMeans extends DisplayClustering {
+
+  DisplaySpectralKMeans() {
+    initialize();
+    this.setTitle("Spectral k-Means Clusters (>" + (int) (significance * 100) + "% of
population)");
+  }
+
+  public static void main(String[] args) throws Exception {
+    DistanceMeasure measure = new ManhattanDistanceMeasure();
+    Path samples = new Path("samples");
+    Path output = new Path("output");
+    HadoopUtil.overwriteOutput(samples);
+    HadoopUtil.overwriteOutput(output);
+
+    RandomUtils.useTestSeed();
+    DisplayClustering.generateSamples();
+    writeSampleData(samples);
+    int maxIter = 10;
+    double convergenceDelta = 0.001;
+    Path affinities = new Path(output, "affinities");
+    Configuration conf = new Configuration();
+    FileSystem fs = FileSystem.get(output.toUri(), conf);
+    if (!fs.exists(output)) {
+      fs.mkdirs(output);
+    }
+    FileWriter writer = new FileWriter(affinities.toString());
+    PrintWriter out = new PrintWriter(writer);
+    try {
+      for (int i = 0; i < SAMPLE_DATA.size(); i++) {
+        for (int j = 0; j < SAMPLE_DATA.size(); j++) {
+          out.println(i + "," + j + "," + measure.distance(SAMPLE_DATA.get(i).get(), SAMPLE_DATA.get(j).get()));
+        }
+      }
+    } finally {
+      out.close();
+    }
+    SpectralKMeansDriver.run(new Configuration(), affinities, output, 1100, 5, measure, convergenceDelta,
maxIter);
+    loadClusters(output);
+    new DisplaySpectralKMeans();
+  }
+
+  // Override the paint() method
+  @Override
+  public void paint(Graphics g) {
+    plotSampleData((Graphics2D) g);
+    plotClusters((Graphics2D) g);
+  }
+}

Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/README.txt
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/README.txt?rev=1006367&r1=1006366&r2=1006367&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/README.txt (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/README.txt Mon
Oct 11 01:03:06 2010
@@ -12,9 +12,9 @@ DisplayClustering - generates 1000 sampl
   * DisplayFuzzyKMeans - uses Fuzzy k-Means clustering
   * DisplayMeanShift - uses MeanShift clustering
   
-  * NOTE: some of these programs display the sample points and then superimposes all of the
clusters
+  * NOTE: some of these programs display the sample points and then superimpose all of the
clusters
     from each iteration. The last iteration's clusters are in bold red and the previous several
are 
-    colored (orange, yellow, green, blue, magenta) in order after which all earlier clusters
are in
+    colored (orange, yellow, green, blue, violet) in order after which all earlier clusters
are in
     light grey. This helps to visualize how the clusters converge upon a solution over multiple
     iterations.
   * NOTE: by changing the parameter values (k, ALPHA_0, numIterations) and the display SIGNIFICANCE



Mime
View raw message