incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r773793 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/java/org/apache/hama/ src/java/org/apache/hama/algebra/ src/java/org/apache/hama/graph/ src/test/org/apache/hama/ src/test/org/apache/hama/examples/
Date Tue, 12 May 2009 05:49:34 GMT
Author: edwardyoon
Date: Tue May 12 05:49:34 2009
New Revision: 773793

URL: http://svn.apache.org/viewvc?rev=773793&view=rev
Log:
When we construct the matrix, dimensions should be defined.

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/examples/org/apache/hama/examples/FileMatrixBlockMult.java
    incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
    incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/MatrixNormMapRed.java
    incubator/hama/trunk/src/java/org/apache/hama/graph/SparseGraph.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
    incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java
    incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java
    incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue May 12 05:49:34 2009
@@ -110,6 +110,8 @@
 
   BUG FIXES
 
+    HAMA-183: When we construct the matrix, 
+                dimensions should be defined. (edwardyoon)
     HAMA-172: Vector.add(Vector v) throw the Index out of bounds 
                 exception when they are different (edwardyoon)
     HAMA-169: Vector's first location bug fixed,

Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/FileMatrixBlockMult.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/FileMatrixBlockMult.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/FileMatrixBlockMult.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/FileMatrixBlockMult.java Tue
May 12 05:49:34 2009
@@ -24,6 +24,10 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.MapWritable;
@@ -33,10 +37,10 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hama.Constants;
 import org.apache.hama.DenseMatrix;
 import org.apache.hama.DenseVector;
 import org.apache.hama.HamaConfiguration;
-import org.apache.hama.Matrix;
 import org.apache.hama.algebra.BlockMultiplyMap;
 import org.apache.hama.algebra.BlockMultiplyReduce;
 import org.apache.hama.io.BlockID;
@@ -44,6 +48,7 @@
 import org.apache.hama.mapred.CollectBlocksMap;
 import org.apache.hama.mapred.CollectBlocksMapReduceBase;
 import org.apache.hama.util.JobManager;
+import org.apache.hama.util.RandomVariable;
 
 public class FileMatrixBlockMult extends AbstractExample {
   final static Log LOG = LogFactory.getLog(FileMatrixBlockMult.class.getName());
@@ -96,16 +101,20 @@
    */
   private static DenseMatrix matMult(Path a, Path b) throws IOException {
     HamaConfiguration conf = new HamaConfiguration();
-    Matrix collectionTable = new DenseMatrix(conf);
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    String collectionTable = "collect_" + RandomVariable.randMatrixPath();
+    HTableDescriptor desc = new HTableDescriptor(collectionTable);
+    desc.addFamily(new HColumnDescriptor(Bytes.toBytes(Constants.BLOCK)));
+    admin.createTable(desc);
+    
+    collectBlocksFromFile(a, true, collectionTable, conf);
+    collectBlocksFromFile(b, false, collectionTable, conf);
 
-    collectBlocksFromFile(a, true, collectionTable.getPath(), conf);
-    collectBlocksFromFile(b, false, collectionTable.getPath(), conf);
-
-    DenseMatrix result = new DenseMatrix(conf);
+    DenseMatrix result = new DenseMatrix(conf, ROWS, COLUMNS);
     JobConf jobConf = new JobConf(conf);
     jobConf.setJobName("multiplication MR job : " + result.getPath());
 
-    BlockMultiplyMap.initJob(collectionTable.getPath(), BlockMultiplyMap.class,
+    BlockMultiplyMap.initJob(collectionTable, BlockMultiplyMap.class,
         BlockID.class, BlockWritable.class, jobConf);
     BlockMultiplyReduce.initJob(result.getPath(), BlockMultiplyReduce.class,
         jobConf);

Modified: incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/AbstractMatrix.java Tue May 12 05:49:34
2009
@@ -354,24 +354,6 @@
     table.commit(update.getBatchUpdate());
   }
   
-  public void increaseRows() throws IOException {
-    int nValue = this.getRows() + 1;
-    try {
-      this.setDimension(nValue, this.getColumns());
-    } catch (NullPointerException ne) {
-      // If there is no metadata of dimension, nothing to do.
-    }
-  }
-  
-  public void increaseColumns() throws IOException {
-    int nValue = this.getColumns() + 1;
-    try {
-      this.setDimension(this.getRows(), nValue);
-    } catch (NullPointerException ne) {
-      // If there is no metadata of dimension, nothing to do.
-    }
-  }
-  
   /** {@inheritDoc} */
   public void add(int i, int j, double value) throws IOException {
     VectorUpdate update = new VectorUpdate(i);
@@ -532,9 +514,9 @@
   public Matrix transpose() throws IOException {
     Matrix result;
     if(this.getType().equals("SparseMatrix")) {
-      result = new SparseMatrix(config);
+      result = new SparseMatrix(config, this.getRows(), this.getColumns());
     } else {
-      result = new DenseMatrix(config);
+      result = new DenseMatrix(config, this.getRows(), this.getColumns());
     }
     
     JobConf jobConf = new JobConf(config);
@@ -549,7 +531,6 @@
         TransposeReduce.class, jobConf);
 
     JobManager.execute(jobConf);
-    result.setDimension(this.getRows(), this.getColumns());
     return result;
   }
   

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue May 12 05:49:34 2009
@@ -25,7 +25,9 @@
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Scanner;
 import org.apache.hadoop.hbase.io.Cell;
@@ -68,19 +70,20 @@
       + "_TMP_dir");
 
   /**
-   * Construct a raw matrix. Just create a table in HBase, but didn't lay any
-   * schema ( such as dimensions: i, j ) on it.
+   * Construct a raw matrix. Just create a table in HBase.
    * 
    * @param conf configuration object
+   * @param m the number of rows.
+   * @param n the number of columns.
    * @throws IOException throw the exception to let the user know what happend,
    *                 if we didn't create the matrix successfully.
    */
-  public DenseMatrix(HamaConfiguration conf) throws IOException {
+  public DenseMatrix(HamaConfiguration conf, int m, int n) throws IOException {
     setConfiguration(conf);
 
     tryToCreateTable(TABLE_PREFIX);
-
     closed = false;
+    this.setDimension(m, n);
   }
 
   /**
@@ -194,7 +197,7 @@
    */
   public static DenseMatrix random(HamaConfiguration conf, int m, int n)
       throws IOException {
-    DenseMatrix rand = new DenseMatrix(conf);
+    DenseMatrix rand = new DenseMatrix(conf, m, n);
     DenseVector vector = new DenseVector();
     LOG.info("Create the " + m + " * " + n + " random matrix : "
         + rand.getPath());
@@ -207,7 +210,6 @@
       rand.setRow(i, vector);
     }
 
-    rand.setDimension(m, n);
     return rand;
   }
 
@@ -222,10 +224,9 @@
    */
   public static DenseMatrix random_mapred(HamaConfiguration conf, int m, int n)
       throws IOException {
-    DenseMatrix rand = new DenseMatrix(conf);
+    DenseMatrix rand = new DenseMatrix(conf, m, n);
     LOG.info("Create the " + m + " * " + n + " random matrix : "
         + rand.getPath());
-    rand.setDimension(m, n);
 
     JobConf jobConf = new JobConf(conf);
     jobConf.setJobName("random matrix MR job : " + rand.getPath());
@@ -285,7 +286,7 @@
    */
   public static DenseMatrix identity(HamaConfiguration conf, int m, int n)
       throws IOException {
-    DenseMatrix identity = new DenseMatrix(conf);
+    DenseMatrix identity = new DenseMatrix(conf, m, n);
     LOG.info("Create the " + m + " * " + n + " identity matrix : "
         + identity.getPath());
 
@@ -297,7 +298,6 @@
       identity.setRow(i, vector);
     }
 
-    identity.setDimension(m, n);
     return identity;
   }
 
@@ -356,6 +356,8 @@
 
   /** {@inheritDoc} */
   public void set(int i, int j, double value) throws IOException {
+    if(this.getRows() < i || this.getColumns() < j)
+      throw new ArrayIndexOutOfBoundsException(i +", "+j);
     VectorUpdate update = new VectorUpdate(i);
     update.put(j, value);
     table.commit(update.getBatchUpdate());
@@ -369,8 +371,8 @@
    * @throws IOException
    */
   public void setRow(int row, Vector vector) throws IOException {
-    if (this.getRows() < row)
-      increaseRows();
+    if (this.getRows() < row || this.getColumns() < vector.size())
+      throw new ArrayIndexOutOfBoundsException(row);
 
     VectorUpdate update = new VectorUpdate(row);
     update.putAll(vector.getEntries());
@@ -385,8 +387,8 @@
    * @throws IOException
    */
   public void setColumn(int column, Vector vector) throws IOException {
-    if (this.getColumns() < column)
-      increaseColumns();
+    if (this.getColumns() < column || this.getRows() < vector.size())
+      throw new ArrayIndexOutOfBoundsException(column);
 
     for (Map.Entry<Writable, Writable> e : vector.getEntries().entrySet()) {
       int key = ((IntWritable) e.getKey()).get();
@@ -408,7 +410,8 @@
   public DenseMatrix add(double alpha, Matrix B) throws IOException {
     ensureForAddition(B);
 
-    DenseMatrix result = new DenseMatrix(config);
+    DenseMatrix result = new DenseMatrix(config, this.getRows(), this
+        .getColumns());
     JobConf jobConf = new JobConf(config);
     jobConf.setJobName("addition MR job" + result.getPath());
 
@@ -421,7 +424,7 @@
     RowCyclicAdditionReduce.initJob(result.getPath(),
         RowCyclicAdditionReduce.class, jobConf);
 
-    JobManager.execute(jobConf, result);
+    JobManager.execute(jobConf);
     return result;
   }
 
@@ -442,7 +445,8 @@
       ensureForAddition(m);
     }
 
-    DenseMatrix result = new DenseMatrix(config);
+    DenseMatrix result = new DenseMatrix(config, this.getRows(), this
+        .getColumns());
     JobConf jobConf = new JobConf(config);
     jobConf.setJobName("addition MR job" + result.getPath());
 
@@ -466,7 +470,7 @@
     RowCyclicAdditionReduce.initJob(result.getPath(),
         RowCyclicAdditionReduce.class, jobConf);
 
-    JobManager.execute(jobConf, result);
+    JobManager.execute(jobConf);
     return result;
   }
 
@@ -486,7 +490,8 @@
    */
   public DenseMatrix mult(Matrix B) throws IOException {
     ensureForMultiplication(B);
-    DenseMatrix result = new DenseMatrix(config);
+    DenseMatrix result = new DenseMatrix(config, this.getRows(), this
+        .getColumns());
 
     for (int i = 0; i < this.getRows(); i++) {
       JobConf jobConf = new JobConf(config);
@@ -504,7 +509,6 @@
       JobManager.execute(jobConf);
     }
 
-    result.setDimension(this.getRows(), this.getColumns());
     return result;
   }
 
@@ -519,13 +523,17 @@
   public DenseMatrix mult(Matrix B, int blocks) throws IOException {
     ensureForMultiplication(B);
 
-    DenseMatrix collectionTable = new DenseMatrix(config);
+    String collectionTable = "collect_" + RandomVariable.randMatrixPath();
+    HTableDescriptor desc = new HTableDescriptor(collectionTable);
+    desc.addFamily(new HColumnDescriptor(Bytes.toBytes(Constants.BLOCK)));
+    this.admin.createTable(desc);
     LOG.info("Collect Blocks");
 
     collectBlocksMapRed(this.getPath(), collectionTable, blocks, true);
     collectBlocksMapRed(B.getPath(), collectionTable, blocks, false);
 
-    DenseMatrix result = new DenseMatrix(config);
+    DenseMatrix result = new DenseMatrix(config, this.getRows(), this
+        .getColumns());
 
     JobConf jobConf = new JobConf(config);
     jobConf.setJobName("multiplication MR job : " + result.getPath());
@@ -533,13 +541,13 @@
     jobConf.setNumMapTasks(config.getNumMapTasks());
     jobConf.setNumReduceTasks(config.getNumReduceTasks());
 
-    BlockMultiplyMap.initJob(collectionTable.getPath(), BlockMultiplyMap.class,
+    BlockMultiplyMap.initJob(collectionTable, BlockMultiplyMap.class,
         BlockID.class, BlockWritable.class, jobConf);
     BlockMultiplyReduce.initJob(result.getPath(), BlockMultiplyReduce.class,
         jobConf);
 
-    JobManager.execute(jobConf, result);
-    // Should be collectionTable removed?
+    JobManager.execute(jobConf);
+    hamaAdmin.delete(collectionTable);
     return result;
   }
 
@@ -634,15 +642,13 @@
    * @param bool
    * @throws IOException
    */
-  public void collectBlocksMapRed(String path, Matrix collectionTable,
+  public void collectBlocksMapRed(String path, String collectionTable,
       int blockNum, boolean bool) throws IOException {
     double blocks = Math.pow(blockNum, 0.5);
     if (!String.valueOf(blocks).endsWith(".0"))
       throw new IOException("can't divide.");
 
     int block_size = (int) blocks;
-    collectionTable.setDimension(block_size, block_size);
-
     JobConf jobConf = new JobConf(config);
     jobConf.setJobName("Blocking MR job" + getPath());
 
@@ -654,8 +660,8 @@
 
     FileInputFormat.addInputPaths(jobConf, path);
 
-    CollectBlocksMapper.initJob(collectionTable.getPath(), bool, block_size,
-        this.getRows(), this.getColumns(), jobConf);
+    CollectBlocksMapper.initJob(collectionTable, bool, block_size, this
+        .getRows(), this.getColumns(), jobConf);
 
     JobManager.execute(jobConf);
   }

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Tue May 12 05:49:34 2009
@@ -68,7 +68,7 @@
 
     this.entries.put(new IntWritable(index), new DoubleEntry(value));
   }
-  
+
   /**
    * Sets the vector
    * 

Modified: incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/SparseMatrix.java Tue May 12 05:49:34 2009
@@ -50,12 +50,12 @@
       .getSimpleName()
       + "_TMP_dir");
   
-  public SparseMatrix(HamaConfiguration conf) throws IOException {
+  public SparseMatrix(HamaConfiguration conf, int m, int n) throws IOException {
     setConfiguration(conf);
 
     tryToCreateTable(TABLE_PREFIX);
-
     closed = false;
+    this.setDimension(m, n);
   }
 
   /**
@@ -91,7 +91,7 @@
    */
   public static SparseMatrix random(HamaConfiguration conf, int m, int n)
       throws IOException {
-    SparseMatrix rand = new SparseMatrix(conf);
+    SparseMatrix rand = new SparseMatrix(conf, m, n);
     SparseVector vector = new SparseVector();
     LOG.info("Create the " + m + " * " + n + " random matrix : "
         + rand.getPath());
@@ -106,15 +106,13 @@
       rand.setRow(i, vector);
     }
 
-    rand.setDimension(m, n);
     return rand;
   }
   
   public static SparseMatrix random_mapred(HamaConfiguration conf, int m, int n) throws IOException
{
-    SparseMatrix rand = new SparseMatrix(conf);
+    SparseMatrix rand = new SparseMatrix(conf, m, n);
     LOG.info("Create the " + m + " * " + n + " random matrix : "
         + rand.getPath());
-    rand.setDimension(m, n);
 
     JobConf jobConf = new JobConf(conf);
     jobConf.setJobName("random matrix MR job : " + rand.getPath());
@@ -225,7 +223,7 @@
    * @throws IOException
    */
   public SparseMatrix mult(Matrix B) throws IOException {
-    SparseMatrix result = new SparseMatrix(config);
+    SparseMatrix result = new SparseMatrix(config, this.getRows(), this.getColumns());
 
     for(int i = 0; i < this.getRows(); i++) {
       JobConf jobConf = new JobConf(config);
@@ -241,7 +239,6 @@
       JobManager.execute(jobConf);
     }
 
-    result.setDimension(this.getRows(), this.getColumns());
     return result;
   }
 
@@ -278,7 +275,7 @@
   @Override
   public void setRow(int row, Vector vector) throws IOException {
     if(this.getRows() < row)
-      increaseRows();
+      throw new ArrayIndexOutOfBoundsException(row);
     
     if(vector.size() > 0) {  // stores if size > 0
       VectorUpdate update = new VectorUpdate(row);

Modified: incubator/hama/trunk/src/java/org/apache/hama/algebra/MatrixNormMapRed.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MatrixNormMapRed.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/MatrixNormMapRed.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/MatrixNormMapRed.java Tue May 12
05:49:34 2009
@@ -43,19 +43,22 @@
 
 /** A Catalog class collect all the mr classes to compute the matrix's norm */
 public class MatrixNormMapRed {
-  
-  /** 
-   * Initialize the job to compute the matrix's norm 
-   * @param inputMatrixPath the input matrix's path 
-   * @param outputPath the output file's name that records the norm of the matrix
+
+  /**
+   * Initialize the job to compute the matrix's norm
+   * 
+   * @param inputMatrixPath the input matrix's path
+   * @param outputPath the output file's name that records the norm of the
+   *                matrix
    * @param mapper Mapper
    * @param combiner Combiner
    * @param reducer Reducer
    * @param jobConf Configuration of the job
    */
   public static void initJob(String inputMatrixPath, String outputPath,
-      Class <? extends MatrixNormMapper> mapper, Class <? extends MatrixNormReducer>
combiner,
-      Class <? extends MatrixNormReducer> reducer, JobConf jobConf) {
+      Class<? extends MatrixNormMapper> mapper,
+      Class<? extends MatrixNormReducer> combiner,
+      Class<? extends MatrixNormReducer> reducer, JobConf jobConf) {
     jobConf.setMapperClass(mapper);
     jobConf.setMapOutputKeyClass(IntWritable.class);
     jobConf.setMapOutputValueClass(DoubleWritable.class);
@@ -63,7 +66,7 @@
     jobConf.setReducerClass(reducer);
     jobConf.setOutputKeyClass(IntWritable.class);
     jobConf.setOutputValueClass(DoubleWritable.class);
-    
+
     // input
     jobConf.setInputFormat(VectorInputFormat.class);
     jobConf.set(VectorInputFormat.COLUMN_LIST, Constants.COLUMN);
@@ -72,27 +75,28 @@
     jobConf.setOutputFormat(SequenceFileOutputFormat.class);
     FileOutputFormat.setOutputPath(jobConf, new Path(outputPath));
   }
-  
+
   /** the interface of norm mapper */
-  public static interface MatrixNormMapper 
-  extends Mapper<IntWritable, MapWritable, IntWritable, DoubleWritable> {
+  public static interface MatrixNormMapper extends
+      Mapper<IntWritable, MapWritable, IntWritable, DoubleWritable> {
     IntWritable nKey = new IntWritable(-1);
     DoubleWritable nValue = new DoubleWritable(0);
   }
-  
+
   /** the interface of norm reducer/combiner */
-  public static interface MatrixNormReducer
-  extends Reducer<IntWritable, DoubleWritable, IntWritable, DoubleWritable> {
+  public static interface MatrixNormReducer extends
+      Reducer<IntWritable, DoubleWritable, IntWritable, DoubleWritable> {
     IntWritable nKey = new IntWritable(-1);
     DoubleWritable nValue = new DoubleWritable(0);
   }
-  
-  ///
-  /// Infinity Norm
-  ///
-  
+
+  // /
+  // / Infinity Norm
+  // /
+
   /** Infinity Norm */
-  public static class MatrixInfinityNormMapper extends MapReduceBase implements MatrixNormMapper
{
+  public static class MatrixInfinityNormMapper extends MapReduceBase implements
+      MatrixNormMapper {
 
     @Override
     public void map(IntWritable key, MapWritable value,
@@ -100,20 +104,21 @@
         throws IOException {
 
       double rowSum = 0;
-      for(Map.Entry<Writable, Writable> e : value.entrySet()) {
+      for (Map.Entry<Writable, Writable> e : value.entrySet()) {
         rowSum += Math.abs(((DoubleEntry) e.getValue()).getValue());
       }
       nValue.set(rowSum);
-      
+
       output.collect(nKey, nValue);
     }
 
   }
-  
+
   /**
    * Matrix Infinity Norm Reducer
    */
-  public static class MatrixInfinityNormReducer extends MapReduceBase implements MatrixNormReducer
{
+  public static class MatrixInfinityNormReducer extends MapReduceBase implements
+      MatrixNormReducer {
 
     private double max = 0;
 
@@ -126,37 +131,39 @@
         max = Math.max(values.next().get(), max);
       }
 
-      // Note: Tricky here. As we known, we collect each row's sum with key(-1). 
-      //       the reduce will just iterate through one key (-1)
-      //       so we collect the max sum-value here
+      // Note: Tricky here. As we known, we collect each row's sum with key(-1).
+      // the reduce will just iterate through one key (-1)
+      // so we collect the max sum-value here
       nValue.set(max);
       output.collect(nKey, nValue);
     }
-    
+
   }
-  
-  ///
-  /// One Norm
-  ///
-  
+
+  // /
+  // / One Norm
+  // /
+
   /** One Norm Mapper */
-  public static class MatrixOneNormMapper extends MapReduceBase implements MatrixNormMapper
{
-    
+  public static class MatrixOneNormMapper extends MapReduceBase implements
+      MatrixNormMapper {
+
     @Override
     public void map(IntWritable key, MapWritable value,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
         throws IOException {
-      
-      for(Map.Entry<Writable, Writable> e : value.entrySet()) {
+
+      for (Map.Entry<Writable, Writable> e : value.entrySet()) {
         nValue.set(((DoubleEntry) e.getValue()).getValue());
-        output.collect((IntWritable)e.getKey(), nValue);
+        output.collect((IntWritable) e.getKey(), nValue);
       }
     }
   }
-  
-  /** One Norm Combiner **/
-  public static class MatrixOneNormCombiner extends MapReduceBase implements MatrixNormReducer
{
-    
+
+  /** One Norm Combiner * */
+  public static class MatrixOneNormCombiner extends MapReduceBase implements
+      MatrixNormReducer {
+
     @Override
     public void reduce(IntWritable key, Iterator<DoubleWritable> values,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
@@ -170,19 +177,20 @@
       output.collect(key, nValue);
     }
   }
-  
-  /** One Norm Reducer **/
-  public static class MatrixOneNormReducer extends MapReduceBase implements MatrixNormReducer
{
+
+  /** One Norm Reducer * */
+  public static class MatrixOneNormReducer extends MapReduceBase implements
+      MatrixNormReducer {
     private double max = 0;
     private Path outDir;
     private JobConf conf;
-    
+
     @Override
     public void configure(JobConf job) {
       outDir = FileOutputFormat.getOutputPath(job);
       conf = job;
     }
-    
+
     @Override
     public void reduce(IntWritable key, Iterator<DoubleWritable> values,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
@@ -191,46 +199,50 @@
       while (values.hasNext()) {
         colSum += values.next().get();
       }
-      
+
       max = Math.max(Math.abs(colSum), max);
     }
-    
+
     @Override
     public void close() throws IOException {
       // write output to a file
       Path outFile = new Path(outDir, "reduce-out");
       FileSystem fileSys = FileSystem.get(conf);
       SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
-          outFile, IntWritable.class, DoubleWritable.class, CompressionType.NONE);
+          outFile, IntWritable.class, DoubleWritable.class,
+          CompressionType.NONE);
       writer.append(new IntWritable(-1), new DoubleWritable(max));
       writer.close();
     }
   }
-  
-  ///
-  /// Frobenius Norm
-  ///
-  
+
+  // /
+  // / Frobenius Norm
+  // /
+
   /** Frobenius Norm Mapper */
-  public static class MatrixFrobeniusNormMapper extends MapReduceBase implements MatrixNormMapper
{
+  public static class MatrixFrobeniusNormMapper extends MapReduceBase implements
+      MatrixNormMapper {
     @Override
     public void map(IntWritable key, MapWritable value,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
         throws IOException {
       double rowSqrtSum = 0;
-      for(Map.Entry<Writable, Writable> e : value.entrySet()) {
-        double cellValue = ((DoubleEntry)e.getValue()).getValue();
-        rowSqrtSum += ( cellValue * cellValue );
+      for (Map.Entry<Writable, Writable> e : value.entrySet()) {
+        double cellValue = ((DoubleEntry) e.getValue()).getValue();
+        rowSqrtSum += (cellValue * cellValue);
       }
-      
+
       nValue.set(rowSqrtSum);
       output.collect(nKey, nValue);
     }
   }
 
   /** Frobenius Norm Combiner */
-  public static class MatrixFrobeniusNormCombiner extends MapReduceBase implements MatrixNormReducer
{
+  public static class MatrixFrobeniusNormCombiner extends MapReduceBase
+      implements MatrixNormReducer {
     private double sqrtSum = 0;
+
     @Override
     public void reduce(IntWritable key, Iterator<DoubleWritable> values,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
@@ -238,17 +250,19 @@
       while (values.hasNext()) {
         sqrtSum += values.next().get();
       }
-      // Note: Tricky here. As we known, we collect each row's sum with key(-1). 
-      //       the reduce will just iterate through one key (-1)
-      //       so we collect the max sum-value here      
+      // Note: Tricky here. As we known, we collect each row's sum with key(-1).
+      // the reduce will just iterate through one key (-1)
+      // so we collect the max sum-value here
       nValue.set(sqrtSum);
       output.collect(nKey, nValue);
     }
   }
-  
+
   /** Frobenius Norm Reducer */
-  public static class MatrixFrobeniusNormReducer extends MapReduceBase implements MatrixNormReducer
{
+  public static class MatrixFrobeniusNormReducer extends MapReduceBase
+      implements MatrixNormReducer {
     private double sqrtSum = 0;
+
     @Override
     public void reduce(IntWritable key, Iterator<DoubleWritable> values,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
@@ -257,39 +271,42 @@
         sqrtSum += values.next().get();
       }
 
-      // Note: Tricky here. As we known, we collect each row's sum with key(-1). 
-      //       the reduce will just iterate through one key (-1)
-      //       so we collect the max sum-value here
+      // Note: Tricky here. As we known, we collect each row's sum with key(-1).
+      // the reduce will just iterate through one key (-1)
+      // so we collect the max sum-value here
       nValue.set(Math.sqrt(sqrtSum));
       output.collect(nKey, nValue);
     }
   }
-  
-  ///
-  /// MaxValue Norm
-  /// 
-  
-  /** MaxValue Norm Mapper **/
-  public static class MatrixMaxValueNormMapper extends MapReduceBase implements MatrixNormMapper
{
+
+  // /
+  // / MaxValue Norm
+  // /
+
+  /** MaxValue Norm Mapper * */
+  public static class MatrixMaxValueNormMapper extends MapReduceBase implements
+      MatrixNormMapper {
     @Override
     public void map(IntWritable key, MapWritable value,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
         throws IOException {
       double max = 0;
-      for(Map.Entry<Writable, Writable> e : value.entrySet()) {
-        double cellValue = Math.abs(((DoubleEntry)e.getValue()).getValue());
+      for (Map.Entry<Writable, Writable> e : value.entrySet()) {
+        double cellValue = Math.abs(((DoubleEntry) e.getValue()).getValue());
         max = cellValue > max ? cellValue : max;
       }
-      
+
       nValue.set(max);
       output.collect(nKey, nValue);
     }
-    
+
   }
-  
+
   /** MaxValue Norm Reducer */
-  public static class MatrixMaxValueNormReducer extends MapReduceBase implements MatrixNormReducer
{
+  public static class MatrixMaxValueNormReducer extends MapReduceBase implements
+      MatrixNormReducer {
     private double max = 0;
+
     @Override
     public void reduce(IntWritable key, Iterator<DoubleWritable> values,
         OutputCollector<IntWritable, DoubleWritable> output, Reporter reporter)
@@ -298,9 +315,9 @@
         max = Math.max(values.next().get(), max);
       }
 
-      // Note: Tricky here. As we known, we collect each row's sum with key(-1). 
-      //       the reduce will just iterate through one key (-1)
-      //       so we collect the max sum-value here
+      // Note: Tricky here. As we known, we collect each row's sum with key(-1).
+      // the reduce will just iterate through one key (-1)
+      // so we collect the max sum-value here
       nValue.set(max);
       output.collect(nKey, nValue);
     }

Modified: incubator/hama/trunk/src/java/org/apache/hama/graph/SparseGraph.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/graph/SparseGraph.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/graph/SparseGraph.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/graph/SparseGraph.java Tue May 12 05:49:34
2009
@@ -51,7 +51,7 @@
    * @throws IOException
    */
   public SparseGraph(HamaConfiguration conf) throws IOException {
-    this.adjMatrix = new SparseMatrix(conf);
+    this.adjMatrix = new SparseMatrix(conf, Integer.MAX_VALUE, Integer.MAX_VALUE);
     this.name = adjMatrix.getPath();
     this.table = adjMatrix.getHTable();
   }

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseMatrix.java Tue May 12 05:49:34
2009
@@ -43,6 +43,7 @@
   private static Matrix m1;
   private static Matrix m2;
   private static Matrix m3;
+  private static Matrix m4;
   private final static String aliase1 = "matrix_aliase_A";
   private final static String aliase2 = "matrix_aliase_B";
   private static HamaConfiguration conf;
@@ -62,6 +63,7 @@
         m1 = DenseMatrix.random(hCluster.getConf(), SIZE, SIZE);
         m2 = DenseMatrix.random(hCluster.getConf(), SIZE, SIZE);
         m3 = DenseMatrix.random(hCluster.getConf(), SIZE, SIZE);
+        m4 = DenseMatrix.random(hCluster.getConf(), SIZE-2, SIZE-2);
       }
 
       protected void tearDown() {
@@ -206,7 +208,7 @@
   }
 
   public void testSetMatrix() throws IOException {
-    Matrix a = new DenseMatrix(conf);
+    Matrix a = new DenseMatrix(conf, m1.getRows(), m1.getColumns());
     a.set(m1);
 
     for (int i = 0; i < 5; i++) {
@@ -218,7 +220,7 @@
   }
 
   public void testSetAlphaMatrix() throws IOException {
-    Matrix a = new DenseMatrix(conf);
+    Matrix a = new DenseMatrix(conf, m1.getRows(), m1.getColumns());
     a.set(0.5, m1);
     
     for (int i = 0; i < 5; i++) {
@@ -271,16 +273,14 @@
       v.set(i, entries[i]);
     }
 
-    m1.setRow(SIZE + 1, v);
-    Iterator<Writable> it = m1.getRow(SIZE + 1).iterator();
+    m1.setRow(SIZE, v);
+    Iterator<Writable> it = m1.getRow(SIZE).iterator();
 
     int i = 0;
     while (it.hasNext()) {
       assertEquals(entries[i], ((DoubleEntry) it.next()).getValue());
       i++;
     }
-    
-    assertEquals(m1.getRows(), SIZE + 1);
   }
 
   public void testSetColumn() throws IOException {
@@ -291,21 +291,19 @@
       v.set(i, entries[i]);
     }
 
-    m1.setColumn(SIZE + 1, v);
-    Iterator<Writable> it = m1.getColumn(SIZE + 1).iterator();
+    m1.setColumn(SIZE, v);
+    Iterator<Writable> it = m1.getColumn(SIZE).iterator();
 
     int i = 0;
     while (it.hasNext()) {
       assertEquals(entries[i], ((DoubleEntry) it.next()).getValue());
       i++;
     }
-    
-    assertEquals(m1.getColumns(), SIZE + 1);
   }
   
   public void testEnsureForAddition() {
     try {
-      m1.add(m2);
+      m1.add(m4);
       fail("Matrix-Addition should be failed while rows and columns aren't same.");
     } catch (IOException e) {
       LOG.info(e.toString());
@@ -314,7 +312,7 @@
 
   public void testEnsureForMultiplication() {
     try {
-      m1.mult(m2);
+      m1.mult(m4);
       fail("Matrix-Mult should be failed while A.columns!=B.rows.");
     } catch (IOException e) {
       LOG.info(e.toString());

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestDenseVector.java Tue May 12 05:49:34
2009
@@ -52,7 +52,7 @@
         HCluster hCluster = new HCluster();
         hCluster.setUp();
 
-        m1 = new DenseMatrix(hCluster.getConf());
+        m1 = new DenseMatrix(hCluster.getConf(), 2, 4);
 
         for (int i = 0; i < 2; i++)
           for (int j = 0; j < 4; j++)

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestSparseVector.java Tue May 12 05:49:34
2009
@@ -45,7 +45,7 @@
         HCluster hCluster = new HCluster();
         hCluster.setUp();
 
-        m1 = new SparseMatrix(hCluster.getConf());
+        m1 = new SparseMatrix(hCluster.getConf(), 2, 4);
 
         for (int i = 0; i < 2; i++)
           for (int j = 0; j < 4; j++)

Modified: incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java?rev=773793&r1=773792&r2=773793&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java Tue
May 12 05:49:34 2009
@@ -31,6 +31,10 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.MapWritable;
@@ -42,6 +46,7 @@
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hama.Constants;
 import org.apache.hama.DenseMatrix;
 import org.apache.hama.DenseVector;
 import org.apache.hama.HCluster;
@@ -54,13 +59,14 @@
 import org.apache.hama.mapred.CollectBlocksMap;
 import org.apache.hama.mapred.CollectBlocksMapReduceBase;
 import org.apache.hama.util.JobManager;
+import org.apache.hama.util.RandomVariable;
 
 public class TestFileMatrixBlockMult extends TestCase {
   final static Log LOG = LogFactory.getLog(TestFileMatrixBlockMult.class
       .getName());
   private static HamaConfiguration conf;
   private static Path[] path = new Path[2];
-  private static Matrix collectionTable;
+  private static String collectionTable;
 
   public static Test suite() {
     TestSetup setup = new TestSetup(
@@ -70,7 +76,11 @@
         hCluster.setUp();
 
         conf = hCluster.getConf();
-        collectionTable = new DenseMatrix(conf);
+        HBaseAdmin admin = new HBaseAdmin(conf);
+        collectionTable = "collect_" + RandomVariable.randMatrixPath();
+        HTableDescriptor desc = new HTableDescriptor(collectionTable);
+        desc.addFamily(new HColumnDescriptor(Bytes.toBytes(Constants.BLOCK)));
+        admin.createTable(desc);
       }
 
       protected void tearDown() {
@@ -115,14 +125,14 @@
   }
 
   public void testFileMatrixMult() throws IOException {
-    collectBlocksFromFile(path[0], true, collectionTable.getPath(), conf);
-    collectBlocksFromFile(path[1], false, collectionTable.getPath(), conf);
+    collectBlocksFromFile(path[0], true, collectionTable, conf);
+    collectBlocksFromFile(path[1], false, collectionTable, conf);
 
-    Matrix result = new DenseMatrix(conf);
+    Matrix result = new DenseMatrix(conf, 4, 4);
     JobConf jobConf = new JobConf(conf);
     jobConf.setJobName("multiplication MR job : " + result.getPath());
 
-    BlockMultiplyMap.initJob(collectionTable.getPath(), BlockMultiplyMap.class,
+    BlockMultiplyMap.initJob(collectionTable, BlockMultiplyMap.class,
         BlockID.class, BlockWritable.class, jobConf);
     BlockMultiplyReduce.initJob(result.getPath(), BlockMultiplyReduce.class,
         jobConf);



Mime
View raw message