incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r691160 - in /incubator/hama/trunk: ./ src/java/org/apache/hama/ src/java/org/apache/hama/algebra/ src/java/org/apache/hama/io/ src/java/org/apache/hama/util/ src/test/org/apache/hama/
Date Tue, 02 Sep 2008 08:59:52 GMT
Author: edwardyoon
Date: Tue Sep  2 01:59:52 2008
New Revision: 691160

URL: http://svn.apache.org/viewvc?rev=691160&view=rev
Log:
Multiplication

Added:
    incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/src/java/org/apache/hama/Constants.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
    incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
    incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java
    incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java
    incubator/hama/trunk/src/test/org/apache/hama/TestVector.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Sep  2 01:59:52 2008
@@ -4,6 +4,7 @@
 
   NEW FEATURES
     
+    HAMA-13: Matrix multiplication (edwardyoon)
     HAMA-48: Add getColumn(int column) method which returns column vector (edwardyoon)
     HAMA-49: Add iterator() method to vector (edwardyoon)
     HAMA-43: Color Hama Logo (Morakot via chanwit)

Modified: incubator/hama/trunk/src/java/org/apache/hama/Constants.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/Constants.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/Constants.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/Constants.java Tue Sep  2 01:59:52 2008
@@ -23,6 +23,7 @@
  * Some constants used in the Hama
  */
 public class Constants {
+  
   /** 
    * Meta-columnFamily to store the matrix-info 
    */

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Tue Sep  2 01:59:52 2008
@@ -28,12 +28,14 @@
 import org.apache.hadoop.hbase.client.Scanner;
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.io.HbaseMapWritable;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hama.algebra.AdditionMap;
 import org.apache.hama.algebra.AdditionReduce;
+import org.apache.hama.algebra.MultiplicationMap;
+import org.apache.hama.algebra.MultiplicationReduce;
 import org.apache.hama.mapred.DenseMap;
 import org.apache.hama.mapred.MatrixReduce;
 import org.apache.hama.util.Numeric;
@@ -136,13 +138,12 @@
     jobConf.setJobName("addition MR job");
 
     DenseMap.initJob(this.getName(), B.getName(), AdditionMap.class,
-        ImmutableBytesWritable.class, DenseVector.class, jobConf);
+        IntWritable.class, DenseVector.class, jobConf);
     MatrixReduce.initJob(C.getName(), AdditionReduce.class, jobConf);
 
     try {
       JobClient.runJob(jobConf);
     } catch (IOException e) {
-      // TODO Auto-generated catch block
       e.printStackTrace();
     }
 
@@ -173,8 +174,23 @@
   }
 
   public Matrix mult(Matrix B) {
-    // TODO Auto-generated method stub
-    return null;
+    String output = RandomVariable.randMatrixName();
+    Matrix C = new DenseMatrix(config, output);
+
+    JobConf jobConf = new JobConf(config);
+    jobConf.setJobName("multiplication MR job");
+
+    DenseMap.initJob(this.getName(), B.getName(), MultiplicationMap.class,
+        IntWritable.class, DenseVector.class, jobConf);
+    MatrixReduce.initJob(C.getName(), MultiplicationReduce.class, jobConf);
+
+    try {
+      JobClient.runJob(jobConf);
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+    return C;
   }
 
   public Matrix multAdd(double alpha, Matrix B, Matrix C) {

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseVector.java Tue Sep  2 01:59:52 2008
@@ -74,14 +74,21 @@
   }
 
   public Vector add(Vector v2) {
-    HbaseMapWritable<byte[], Cell> trunk = new HbaseMapWritable<byte[], Cell>();
+    if (this.size() == 0) {
+      DenseVector trunk = (DenseVector) v2;
+      this.row = trunk.row;
+      this.cells = trunk.cells;
+      return this;
+    }
+
     for (int i = 0; i < this.size(); i++) {
       double value = (this.get(i) + v2.get(i));
-      Cell cValue = new Cell(String.valueOf(value), System.currentTimeMillis());
-      trunk.put(Numeric.getColumnIndex(i), cValue);
+
+      Cell cValue = new Cell(Numeric.doubleToBytes(value), now());
+      this.cells.put(Numeric.getColumnIndex(i), cValue);
     }
 
-    return new DenseVector(row, trunk);
+    return this;
   }
 
   public double dot(Vector v) {
@@ -99,18 +106,22 @@
     Set<byte[]> keySet = cells.keySet();
     Iterator<byte[]> it = keySet.iterator();
 
+    int i = 0;
     while (it.hasNext()) {
       byte[] key = it.next();
-      double oValue = Numeric.bytesToDouble(get(key).getValue());
+      double oValue = Numeric.bytesToDouble(this.get(key).getValue());
       double nValue = oValue * alpha;
-      Cell cValue = new Cell(String.valueOf(nValue), System.currentTimeMillis());
-      cells.put(key, cValue);
+
+      Cell cValue = new Cell(Numeric.doubleToBytes(nValue), now());
+      this.cells.put(Numeric.getColumnIndex(i), cValue);
+      i++;
     }
 
     return this;
   }
 
   public double get(int index) {
+
     return Numeric.bytesToDouble(cells.get(Numeric.getColumnIndex(index))
         .getValue());
   }
@@ -127,7 +138,7 @@
   }
 
   public void set(int index, double value) {
-    Cell cValue = new Cell(String.valueOf(value), System.currentTimeMillis());
+    Cell cValue = new Cell(Numeric.doubleToBytes(value), now());
     cells.put(Numeric.getColumnIndex(index), cValue);
   }
 

Added: incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java?rev=691160&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationMap.java Tue Sep 
2 01:59:52 2008
@@ -0,0 +1,55 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseVector;
+import org.apache.hama.Vector;
+import org.apache.hama.mapred.DenseMap;
+import org.apache.hama.util.Numeric;
+import org.apache.log4j.Logger;
+
+public class MultiplicationMap extends DenseMap<IntWritable, DenseVector> {
+  static final Logger LOG = Logger.getLogger(MultiplicationMap.class);
+
+  @Override
+  public void map(IntWritable key, DenseVector value,
+      OutputCollector<IntWritable, DenseVector> output, Reporter reporter)
+      throws IOException {
+
+    Iterator<Cell> it = value.iterator();
+    int i = 0;
+    while (it.hasNext()) {
+      Cell c = it.next();
+      
+      Vector v = MATRIX_B.getRow(i);
+      
+      double alpha = Numeric.bytesToDouble(c.getValue());
+      output.collect(key, (DenseVector) v.scale(alpha));
+      i++;
+    }
+  }
+}

Added: incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java?rev=691160&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/MultiplicationReduce.java Tue Sep
 2 01:59:52 2008
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseVector;
+import org.apache.hama.mapred.MatrixReduce;
+import org.apache.hama.util.Numeric;
+import org.apache.log4j.Logger;
+
+public class MultiplicationReduce extends
+    MatrixReduce<IntWritable, DenseVector> {
+  static final Logger LOG = Logger.getLogger(MultiplicationReduce.class);
+
+  @Override
+  public void reduce(IntWritable key, Iterator<DenseVector> values,
+      OutputCollector<IntWritable, BatchUpdate> output, Reporter reporter)
+      throws IOException {
+
+    BatchUpdate b = new BatchUpdate(Numeric.intToBytes(key.get()));
+    DenseVector sum;
+    Map<Integer, Double> buffer = new HashMap<Integer, Double>();
+
+    // Summation vectors
+    while (values.hasNext()) {
+      sum = values.next();
+      for (int i = 0; i < sum.size(); i++) {
+        if (buffer.containsKey(i)) {
+          buffer.put(i, sum.get(i) + buffer.get(i));
+        } else {
+          buffer.put(i, sum.get(i));
+        }
+      }
+    }
+
+    for (Map.Entry<Integer, Double> f : buffer.entrySet()) {
+      byte[] value = Numeric.doubleToBytes(f.getValue());
+      b.put(Numeric.getColumnIndex(f.getKey()), value);
+    }
+
+    output.collect(key, b);
+  }
+
+}

Modified: incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/io/VectorWritable.java Tue Sep  2 01:59:52
2008
@@ -206,4 +206,8 @@
       return cell;
     }
   }
+  
+  public long now() {
+    return System.currentTimeMillis();
+  }
 }

Modified: incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/util/Numeric.java Tue Sep  2 01:59:52 2008
@@ -26,6 +26,7 @@
  * Provides a number format conversion
  */
 public class Numeric {
+  
   /**
    * Bytes to integer conversion
    * 

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestMatrix.java Tue Sep  2 01:59:52 2008
@@ -31,16 +31,14 @@
 public class TestMatrix extends HamaTestCase {
 
   /**
-   * Random matrix creation test
-   * 
-   * @throws IOException
+   * Matrix Test
    * 
    * @throws IOException
    */
-  public void testRandomMatrix() throws IOException {
+  public void testGetMatrixSize() throws IOException {
     Matrix rand = DenseMatrix.random(conf, SIZE, SIZE);
     assertTrue(rand.getRows() == SIZE);
-    
+
     getColumnTest(rand);
   }
 
@@ -59,4 +57,55 @@
       x++;
     }
   }
+
+  /**
+   * Test matrices addition
+   */
+  public void testMatrixAdd() {
+    Matrix rand1 = DenseMatrix.random(conf, SIZE, SIZE);
+    Matrix rand2 = DenseMatrix.random(conf, SIZE, SIZE);
+
+    Matrix result = rand1.add(rand2);
+
+    for (int i = 0; i < SIZE; i++) {
+      for (int j = 0; j < SIZE; j++) {
+        assertEquals(result.get(i, j), rand1.get(i, j) + rand2.get(i, j));
+      }
+    }
+  }
+
+  /**
+   * Test matrices multiplication
+   */
+  public void testMatrixMult() {
+    Matrix m1 = DenseMatrix.random(conf, SIZE, SIZE);
+    Matrix m2 = DenseMatrix.random(conf, SIZE, SIZE);
+
+    Matrix result = m1.mult(m2);
+
+    verifyMultResult(SIZE, m1, m2, result);
+  }
+
+  /**
+   * Verifying multiplication result
+   * 
+   * @param size
+   * @param m1
+   * @param m2
+   * @param result
+   */
+  private void verifyMultResult(int size, Matrix m1, Matrix m2, Matrix result) {
+    double[][] C = new double[SIZE][SIZE];
+
+    for (int i = 0; i < SIZE; i++)
+      for (int j = 0; j < SIZE; j++)
+        for (int k = 0; k < SIZE; k++)
+          C[i][k] += m1.get(i, j) * m2.get(j, k);
+
+    for (int i = 0; i < 2; i++) {
+      for (int j = 0; j < 2; j++) {
+        assertEquals(result.get(i, j), C[i][j]);
+      }
+    }
+  }
 }

Modified: incubator/hama/trunk/src/test/org/apache/hama/TestVector.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/TestVector.java?rev=691160&r1=691159&r2=691160&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/TestVector.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/TestVector.java Tue Sep  2 01:59:52 2008
@@ -34,7 +34,8 @@
 
   /**
    * Test vector
-   * @throws IOException 
+   * 
+   * @throws IOException
    */
   public void testGetVector() throws IOException {
     Matrix m1 = new DenseMatrix(conf, m);
@@ -79,6 +80,11 @@
     assertEquals(norm2, ((DenseVector) v1).getNorm2());
   }
 
+  /**
+   * Test scaling
+   * 
+   * @param v2
+   */
   private void scalingTest(Vector v2) {
     v2.scale(0.5);
 
@@ -87,12 +93,18 @@
     }
   }
 
+  /**
+   * Test get/set methods
+   */
   public void testGetSet() {
     Vector v1 = new DenseVector();
     v1.set(0, 0.2);
     assertEquals(v1.get(0), 0.2);
   }
 
+  /**
+   * Test iterator
+   */
   public void testIterator() {
     Vector v1 = new DenseVector();
     v1.set(0, 0.2);



Mime
View raw message