incubator-hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r711787 - in /incubator/hama/trunk/src: java/org/apache/hama/ java/org/apache/hama/algebra/ test/org/apache/hama/mapred/
Date Thu, 06 Nov 2008 08:00:14 GMT
Author: edwardyoon
Date: Thu Nov  6 00:00:14 2008
New Revision: 711787

URL: http://svn.apache.org/viewvc?rev=711787&view=rev
Log:
I also re-name the map/red classes.

Added:
    incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
Removed:
    incubator/hama/trunk/src/java/org/apache/hama/algebra/Add1DLayoutMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/Add1DLayoutReduce.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/Mult1DLayoutMap.java
    incubator/hama/trunk/src/java/org/apache/hama/algebra/Mult1DLayoutReduce.java
Modified:
    incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
    incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java

Modified: incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java?rev=711787&r1=711786&r2=711787&view=diff
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java (original)
+++ incubator/hama/trunk/src/java/org/apache/hama/DenseMatrix.java Thu Nov  6 00:00:14 2008
@@ -30,10 +30,10 @@
 import org.apache.hadoop.hbase.io.RowResult;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hama.algebra.Add1DLayoutMap;
-import org.apache.hama.algebra.Add1DLayoutReduce;
-import org.apache.hama.algebra.Mult1DLayoutMap;
-import org.apache.hama.algebra.Mult1DLayoutReduce;
+import org.apache.hama.algebra.RowCyclicAdditionMap;
+import org.apache.hama.algebra.RowCyclicAdditionReduce;
+import org.apache.hama.algebra.SIMDMultiplyMap;
+import org.apache.hama.algebra.SIMDMultiplyReduce;
 import org.apache.hama.io.VectorEntry;
 import org.apache.hama.io.VectorMapWritable;
 import org.apache.hama.io.VectorUpdate;
@@ -253,9 +253,9 @@
     jobConf.setNumReduceTasks(Integer.parseInt(config
         .get("mapred.reduce.tasks")));
 
-    Add1DLayoutMap.initJob(this.getPath(), B.getPath(), Add1DLayoutMap.class,
+    RowCyclicAdditionMap.initJob(this.getPath(), B.getPath(), RowCyclicAdditionMap.class,
         IntWritable.class, VectorWritable.class, jobConf);
-    RowCyclicReduce.initJob(result.getPath(), Add1DLayoutReduce.class, jobConf);
+    RowCyclicReduce.initJob(result.getPath(), RowCyclicAdditionReduce.class, jobConf);
 
     JobManager.execute(jobConf, result);
     return result;
@@ -303,9 +303,9 @@
     jobConf.setNumReduceTasks(Integer.parseInt(config
         .get("mapred.reduce.tasks")));
 
-    Mult1DLayoutMap.initJob(this.getPath(), B.getPath(), Mult1DLayoutMap.class,
+    SIMDMultiplyMap.initJob(this.getPath(), B.getPath(), SIMDMultiplyMap.class,
         IntWritable.class, VectorWritable.class, jobConf);
-    RowCyclicReduce.initJob(result.getPath(), Mult1DLayoutReduce.class, jobConf);
+    RowCyclicReduce.initJob(result.getPath(), SIMDMultiplyReduce.class, jobConf);
     JobManager.execute(jobConf, result);
     return result;
   }

Added: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java?rev=711787&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionMap.java Thu Nov
 6 00:00:14 2008
@@ -0,0 +1,73 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseMatrix;
+import org.apache.hama.DenseVector;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.Matrix;
+import org.apache.hama.Vector;
+import org.apache.hama.io.VectorWritable;
+import org.apache.hama.mapred.RowCyclicMap;
+import org.apache.log4j.Logger;
+
+public class RowCyclicAdditionMap extends RowCyclicMap<IntWritable, VectorWritable>
{
+  static final Logger LOG = Logger.getLogger(RowCyclicAdditionMap.class);
+  protected Matrix matrix_b;
+  public static final String MATRIX_B = "hama.addition.matrix.b";
+
+  public void configure(JobConf job) {
+    try {
+      matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
+    } catch (IOException e) {
+      LOG.warn("Load matrix_b failed : " + e.getMessage());
+    }
+  }
+
+  public static void initJob(String matrix_a, String matrix_b,
+      Class<RowCyclicAdditionMap> map, Class<IntWritable> outputKeyClass,
+      Class<VectorWritable> outputValueClass, JobConf jobConf) {
+
+    jobConf.setMapOutputValueClass(outputValueClass);
+    jobConf.setMapOutputKeyClass(outputKeyClass);
+    jobConf.setMapperClass(map);
+    jobConf.set(MATRIX_B, matrix_b);
+
+    initJob(matrix_a, map, jobConf);
+  }
+
+  @Override
+  public void map(IntWritable key, VectorWritable value,
+      OutputCollector<IntWritable, VectorWritable> output, Reporter reporter)
+      throws IOException {
+
+    Vector v1 = matrix_b.getRow(key.get());
+    output.collect(key, new VectorWritable(key.get(), (DenseVector) v1
+        .add(value.getDenseVector())));
+
+  }
+
+}

Added: incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java?rev=711787&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/RowCyclicAdditionReduce.java Thu
Nov  6 00:00:14 2008
@@ -0,0 +1,51 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.io.VectorEntry;
+import org.apache.hama.io.VectorUpdate;
+import org.apache.hama.io.VectorWritable;
+import org.apache.hama.mapred.RowCyclicReduce;
+
+public class RowCyclicAdditionReduce extends RowCyclicReduce<IntWritable, VectorWritable>
{
+
+  @Override
+  public void reduce(IntWritable key, Iterator<VectorWritable> values,
+      OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
+      throws IOException {
+
+    VectorUpdate update = new VectorUpdate(key.get());
+    VectorWritable vector = values.next();
+    
+    for (Map.Entry<Integer, VectorEntry> f : vector.entrySet()) {
+      update.put(f.getKey(), f.getValue().getValue());
+    }
+
+    output.collect(key, update);
+  }
+
+}

Added: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java?rev=711787&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyMap.java Thu Nov  6
00:00:14 2008
@@ -0,0 +1,78 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.DenseMatrix;
+import org.apache.hama.DenseVector;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.Matrix;
+import org.apache.hama.Vector;
+import org.apache.hama.io.VectorWritable;
+import org.apache.hama.mapred.RowCyclicMap;
+import org.apache.log4j.Logger;
+
+/**
+ * SIMD version
+ */
+public class SIMDMultiplyMap extends RowCyclicMap<IntWritable, VectorWritable> {
+  static final Logger LOG = Logger.getLogger(SIMDMultiplyMap.class);
+  protected Matrix matrix_b;
+  public static final String MATRIX_B = "hama.multiplication.matrix.b";
+
+  public void configure(JobConf job) {
+    try {
+      matrix_b = new DenseMatrix(new HamaConfiguration(), job.get(MATRIX_B, ""));
+    } catch (IOException e) {
+      LOG.warn("Load matrix_b failed : " + e.getMessage());
+    }
+  }
+
+  public static void initJob(String matrix_a, String matrix_b,
+      Class<SIMDMultiplyMap> map, Class<IntWritable> outputKeyClass,
+      Class<VectorWritable> outputValueClass, JobConf jobConf) {
+
+    jobConf.setMapOutputValueClass(outputValueClass);
+    jobConf.setMapOutputKeyClass(outputKeyClass);
+    jobConf.setMapperClass(map);
+    jobConf.set(MATRIX_B, matrix_b);
+
+    initJob(matrix_a, map, jobConf);
+  }
+
+  @Override
+  public void map(IntWritable key, VectorWritable value,
+      OutputCollector<IntWritable, VectorWritable> output, Reporter reporter)
+      throws IOException {
+
+    DenseVector v1 = value.getDenseVector();
+    
+    for(int i = 0; i < v1.size(); i++) {
+      Vector v2 = matrix_b.getRow(i);
+      DenseVector sum = (DenseVector) v2.scale(v1.get(i));
+      output.collect(key, new VectorWritable(key.get(), sum));
+    }
+  }
+}

Added: incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java?rev=711787&view=auto
==============================================================================
--- incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java (added)
+++ incubator/hama/trunk/src/java/org/apache/hama/algebra/SIMDMultiplyReduce.java Thu Nov
 6 00:00:14 2008
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.algebra;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hama.io.VectorUpdate;
+import org.apache.hama.io.VectorWritable;
+import org.apache.hama.mapred.RowCyclicReduce;
+import org.apache.log4j.Logger;
+
+public class SIMDMultiplyReduce extends
+    RowCyclicReduce<IntWritable, VectorWritable> {
+  static final Logger LOG = Logger.getLogger(SIMDMultiplyReduce.class);
+  public static final Map<Integer, Double> buffer = new HashMap<Integer, Double>();
+  
+  @Override
+  public void reduce(IntWritable key, Iterator<VectorWritable> values,
+      OutputCollector<IntWritable, VectorUpdate> output, Reporter reporter)
+      throws IOException {
+
+    VectorUpdate update = new VectorUpdate(key.get());
+    VectorWritable sum;
+    
+    // Summation
+    while (values.hasNext()) {
+      sum = values.next();
+      for (int i = 0; i < sum.size(); i++) {
+        if (buffer.containsKey(i)) {
+          buffer.put(i, sum.get(i).getValue() + buffer.get(i));
+        } else {
+          buffer.put(i, sum.get(i).getValue());
+        }
+      }
+    }
+    
+    update.putAll(buffer);
+    buffer.clear();
+    output.collect(key, update);
+  }
+
+}

Modified: incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java?rev=711787&r1=711786&r2=711787&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/mapred/TestMatrixMapReduce.java Thu Nov
 6 00:00:14 2008
@@ -27,8 +27,8 @@
 import org.apache.hama.DenseMatrix;
 import org.apache.hama.HCluster;
 import org.apache.hama.Matrix;
-import org.apache.hama.algebra.Add1DLayoutMap;
-import org.apache.hama.algebra.Add1DLayoutReduce;
+import org.apache.hama.algebra.RowCyclicAdditionMap;
+import org.apache.hama.algebra.RowCyclicAdditionReduce;
 import org.apache.hama.io.VectorWritable;
 import org.apache.log4j.Logger;
 
@@ -69,9 +69,9 @@
     JobConf jobConf = new JobConf(conf, TestMatrixMapReduce.class);
     jobConf.setJobName("test MR job");
 
-    Add1DLayoutMap.initJob(pathA, pathB, Add1DLayoutMap.class, IntWritable.class,
+    RowCyclicAdditionMap.initJob(pathA, pathB, RowCyclicAdditionMap.class, IntWritable.class,
         VectorWritable.class, jobConf);
-    RowCyclicReduce.initJob(output, Add1DLayoutReduce.class, jobConf);
+    RowCyclicReduce.initJob(output, RowCyclicAdditionReduce.class, jobConf);
 
     jobConf.setNumMapTasks(2);
     jobConf.setNumReduceTasks(2);



Mime
View raw message