hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r1420632 - in /hama/trunk: ./ examples/src/main/java/org/apache/hama/examples/ examples/src/main/java/org/apache/hama/examples/util/ examples/src/test/java/org/apache/hama/examples/
Date Wed, 12 Dec 2012 12:09:49 GMT
Author: edwardyoon
Date: Wed Dec 12 12:09:48 2012
New Revision: 1420632

URL: http://svn.apache.org/viewvc?rev=1420632&view=rev
Log:
Add SpMV example

Added:
    hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java
    hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java
    hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java
    hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java
Modified:
    hama/trunk/CHANGES.txt
    hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java

Modified: hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hama/trunk/CHANGES.txt?rev=1420632&r1=1420631&r2=1420632&view=diff
==============================================================================
--- hama/trunk/CHANGES.txt (original)
+++ hama/trunk/CHANGES.txt Wed Dec 12 12:09:48 2012
@@ -4,6 +4,7 @@ Release 0.7 (unreleased changes)
 
   NEW FEATURES
 
+   HAMA-524: Add SpMV example (Mikalai Parafeniuk via edwardyoon)
    HAMA-658: Add random symmetric sparse matrix generator (edwardyoon)
 
   BUG FIXES

Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java (added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java Wed Dec 12 12:09:48
2012
@@ -0,0 +1,330 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.bsp.BSP;
+import org.apache.hama.bsp.BSPJob;
+import org.apache.hama.bsp.BSPJobClient;
+import org.apache.hama.bsp.BSPPeer;
+import org.apache.hama.bsp.ClusterStatus;
+import org.apache.hama.bsp.FileOutputFormat;
+import org.apache.hama.bsp.SequenceFileInputFormat;
+import org.apache.hama.bsp.SequenceFileOutputFormat;
+import org.apache.hama.bsp.sync.SyncException;
+import org.apache.hama.examples.util.DenseVectorWritable;
+import org.apache.hama.examples.util.SparseVectorWritable;
+import org.apache.hama.util.KeyValuePair;
+
+/**
+ * Sparse matrix vector multiplication. Currently it uses row-wise access.
+ * Assumptions: 1) each peer should have copy of input vector for efficient
+ * operations. 2) row-wise implementation is good because we don't need to care
+ * about communication 3) the main way to improve performance - create custom
+ * Partitioner
+ * 
+ * TODO need to be simplified.
+ */
+public class SpMV {
+
+  protected static final Log LOG = LogFactory.getLog(SpMV.class);
+  private static String resultPath;
+  private static final String outputPathString = "spmv.outputpath";
+  private static final String inputMatrixPathString = "spmv.inputmatrixpath";
+  private static final String inputVectorPathString = "spmv.inputvectorpath";
+  private static String requestedBspTasksString = "bsptask.count";
+  private static final String intermediate = "/part";
+
+  enum RowCounter {
+    TOTAL_ROWS
+  }
+
+  public static String getResultPath() {
+    return resultPath;
+  }
+
+  public static void setResultPath(String resultPath) {
+    SpMV.resultPath = resultPath;
+  }
+
+  /**
+   * IMPORTANT: This can be a bottle neck. Problem can be here{@core
+   * WritableUtil.convertSpMVOutputToDenseVector()}
+   */
+  private static void convertToDenseVector(Configuration conf)
+      throws IOException {
+    String resultPath = convertSpMVOutputToDenseVector(
+        conf.get(outputPathString), conf);
+    setResultPath(resultPath);
+  }
+
+  /**
+   * This class performs sparse matrix vector multiplication. u = m * v.
+   */
+  private static class SpMVBSP
+      extends
+      BSP<IntWritable, SparseVectorWritable, IntWritable, DoubleWritable, NullWritable>
{
+    private DenseVectorWritable v;
+
+    /**
+     * Each peer reads input dense vector.
+     */
+    @Override
+    public void setup(
+        BSPPeer<IntWritable, SparseVectorWritable, IntWritable, DoubleWritable, NullWritable>
peer)
+        throws IOException, SyncException, InterruptedException {
+      // reading input vector, which represented as matrix row
+      Configuration conf = peer.getConfiguration();
+      v = new DenseVectorWritable();
+      readFromFile(conf.get(inputVectorPathString), v, conf);
+      peer.sync();
+    }
+
+    /**
+     * Local inner product computation and output.
+     */
+    @Override
+    public void bsp(
+        BSPPeer<IntWritable, SparseVectorWritable, IntWritable, DoubleWritable, NullWritable>
peer)
+        throws IOException, SyncException, InterruptedException {
+      KeyValuePair<IntWritable, SparseVectorWritable> row = null;
+      while ((row = peer.readNext()) != null) {
+        // it will be needed in conversion of output to result vector
+        peer.getCounter(RowCounter.TOTAL_ROWS).increment(1L);
+        int key = row.getKey().get();
+        double sum = 0;
+        SparseVectorWritable mRow = row.getValue();
+        if (v.getSize() != mRow.getSize())
+          throw new RuntimeException("Matrix row with index = " + key
+              + " is not consistent with input vector. Row size = "
+              + mRow.getSize() + " vector size = " + v.getSize());
+        List<Integer> mIndeces = mRow.getIndeces();
+        List<Double> mValues = mRow.getValues();
+        for (int i = 0; i < mIndeces.size(); i++)
+          sum += v.get(mIndeces.get(i)) * mValues.get(i);
+        peer.write(new IntWritable(key), new DoubleWritable(sum));
+      }
+    }
+
+  }
+
+  /**
+   * Method which actually starts SpMV.
+   */
+  private static void startTask(HamaConfiguration conf) throws IOException,
+      InterruptedException, ClassNotFoundException {
+    BSPJob bsp = new BSPJob(conf, SpMV.class);
+    bsp.setJobName("Sparse matrix vector multiplication");
+    bsp.setBspClass(SpMVBSP.class);
+    /*
+     * Input matrix is presented as pairs of integer and SparseVectorWritable.
+     * Output is pairs of integer and double
+     */
+    bsp.setInputFormat(SequenceFileInputFormat.class);
+    bsp.setOutputKeyClass(IntWritable.class);
+    bsp.setOutputValueClass(DoubleWritable.class);
+    bsp.setOutputFormat(SequenceFileOutputFormat.class);
+    bsp.setInputPath(new Path(conf.get(inputMatrixPathString)));
+
+    FileOutputFormat.setOutputPath(bsp, new Path(conf.get(outputPathString)));
+
+    BSPJobClient jobClient = new BSPJobClient(conf);
+    ClusterStatus cluster = jobClient.getClusterStatus(true);
+
+    int requestedTasks = conf.getInt(requestedBspTasksString, -1);
+    if (requestedTasks != -1) {
+      bsp.setNumBspTask(requestedTasks);
+    } else {
+      bsp.setNumBspTask(cluster.getMaxTasks());
+    }
+
+    long startTime = System.currentTimeMillis();
+    if (bsp.waitForCompletion(true)) {
+      LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime)
+          / 1000.0 + " seconds.");
+      convertToDenseVector(conf);
+      LOG.info("Result is in " + getResultPath());
+    } else {
+      setResultPath(null);
+    }
+  }
+
+  private static void printUsage() {
+    LOG.info("Usage: spmv <Matrix> <Vector> <output> [number of tasks (default
max)]");
+  }
+
+  /**
+   * Function parses command line in standart form.
+   */
+  private static void parseArgs(HamaConfiguration conf, String[] args) {
+    if (args.length < 3) {
+      printUsage();
+      System.exit(-1);
+    }
+
+    conf.set(inputMatrixPathString, args[0]);
+    conf.set(inputVectorPathString, args[1]);
+
+    Path path = new Path(args[2]);
+    path = path.suffix(intermediate);
+    conf.set(outputPathString, path.toString());
+
+    if (args.length == 4) {
+      try {
+        int taskCount = Integer.parseInt(args[3]);
+        if (taskCount < 0) {
+          printUsage();
+          throw new IllegalArgumentException(
+              "The number of requested tasks can't be negative. Actual value: "
+                  + String.valueOf(taskCount));
+        }
+        conf.setInt(requestedBspTasksString, taskCount);
+      } catch (NumberFormatException e) {
+        printUsage();
+        throw new IllegalArgumentException(
+            "The format of requested task count is int. Can not parse value: "
+                + args[3]);
+      }
+    }
+  }
+
+  /**
+   * SpMV produces a file, which contains result dense vector in format of pairs
+   * of integer and double. The aim of this method is to convert SpMV output to
+   * format usable in subsequent computation - dense vector. It can be usable
+   * for iterative solvers. IMPORTANT: currently it is used in SpMV. It can be a
+   * bottle neck, because all input needs to be stored in memory.
+   * 
+   * @param SpMVoutputPathString output path, which represents directory with
+   *          part files.
+   * @param conf configuration
+   * @return path to output vector.
+   * @throws IOException
+   */
+  public static String convertSpMVOutputToDenseVector(
+      String SpMVoutputPathString, Configuration conf) throws IOException {
+    List<Integer> indeces = new ArrayList<Integer>();
+    List<Double> values = new ArrayList<Double>();
+
+    FileSystem fs = FileSystem.get(conf);
+    Path SpMVOutputPath = new Path(SpMVoutputPathString);
+    Path resultOutputPath = SpMVOutputPath.getParent().suffix("/result");
+    FileStatus[] stats = fs.listStatus(SpMVOutputPath);
+    for (FileStatus stat : stats) {
+      String filePath = stat.getPath().toUri().getPath();
+      SequenceFile.Reader reader = null;
+      fs.open(new Path(filePath));
+      try {
+        reader = new SequenceFile.Reader(fs, new Path(filePath), conf);
+        IntWritable key = new IntWritable();
+        DoubleWritable value = new DoubleWritable();
+        while (reader.next(key, value)) {
+          indeces.add(key.get());
+          values.add(value.get());
+        }
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      } finally {
+        if (reader != null)
+          reader.close();
+      }
+    }
+    DenseVectorWritable result = new DenseVectorWritable();
+    result.setSize(indeces.size());
+    for (int i = 0; i < indeces.size(); i++)
+      result.addCell(indeces.get(i), values.get(i));
+    writeToFile(resultOutputPath.toString(), result, conf);
+    return resultOutputPath.toString();
+  }
+
+  public static void readFromFile(String pathString, Writable result,
+      Configuration conf) throws IOException {
+    FileSystem fs = FileSystem.get(conf);
+    SequenceFile.Reader reader = null;
+    Path path = new Path(pathString);
+    List<String> filePaths = new ArrayList<String>();
+    if (!fs.isFile(path)) {
+      FileStatus[] stats = fs.listStatus(path);
+      for (FileStatus stat : stats) {
+        filePaths.add(stat.getPath().toUri().getPath());
+      }
+    } else if (fs.isFile(path)) {
+      filePaths.add(path.toString());
+    }
+    try {
+      for (String filePath : filePaths) {
+        reader = new SequenceFile.Reader(fs, new Path(filePath), conf);
+        IntWritable key = new IntWritable();
+        reader.next(key, result);
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    } finally {
+      if (reader != null)
+        reader.close();
+    }
+  }
+
+  /**
+   * This method is used to write vector from memory to specified path.
+   * 
+   * @param pathString output path
+   * @param result instance of vector to be writed
+   * @param conf configuration
+   * @throws IOException
+   */
+  public static void writeToFile(String pathString, Writable result,
+      Configuration conf) throws IOException {
+    FileSystem fs = FileSystem.get(conf);
+    SequenceFile.Writer writer = null;
+    try {
+      writer = new SequenceFile.Writer(fs, conf, new Path(pathString),
+          IntWritable.class, result.getClass());
+      IntWritable key = new IntWritable();
+      writer.append(key, result);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    } finally {
+      if (writer != null)
+        writer.close();
+    }
+  }
+
+  public static void main(String[] args) throws IOException,
+      InterruptedException, ClassNotFoundException {
+    HamaConfiguration conf = new HamaConfiguration();
+    parseArgs(conf, args);
+    startTask(conf);
+  }
+
+}

Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java
(added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java
Wed Dec 12 12:09:48 2012
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents dense vector. It will improve memory consumption up to
+ * two times in comparison to SparseVectorWritable in case of vectors
+ * which sparsity is close to 1. Internally represents vector values as array.
+ * Can be used in SpMV for representation of input and output vector.
+ */
+public class DenseVectorWritable implements Writable {
+
+  private double values[];
+
+  public DenseVectorWritable() {
+    values = new double[0];
+  }
+
+  public int getSize() {
+    return values.length;
+  }
+
+  public void setSize(int size) {
+    values = new double[size];
+  }
+
+  public double get(int index) {
+    return values[index];
+  }
+
+  public void addCell(int index, double value) {
+    values[index] = value;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    int size = in.readInt();
+    int len = in.readInt();
+    setSize(size);
+    for (int i = 0; i < len; i++) {
+      int index = in.readInt();
+      double value = in.readDouble();
+      values[index] = value;
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(getSize());
+    out.writeInt(getSize());
+    for (int i = 0; i < getSize(); i++) {
+      out.writeInt(i);
+      out.writeDouble(values[i]);
+    }
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder st = new StringBuilder();
+    st.append(" "+getSize()+" "+getSize());
+    for (int i = 0; i < getSize(); i++) 
+      st.append(" "+i+" "+values[i]);
+    return st.toString();
+  }
+
+}

Modified: hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java?rev=1420632&r1=1420631&r2=1420632&view=diff
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java (original)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java Wed Dec
12 12:09:48 2012
@@ -24,13 +24,18 @@ public class Generator {
       System.out.println("Valid command names are:");
       System.out
           .println("  symmetric: Generate random symmetric matrix, which can be used as a
input of graph examples.");
+      System.out.println("  square: Generate random square matrix.");
       System.exit(1);
     }
 
+    String[] newArgs = new String[args.length - 1];
+    System.arraycopy(args, 1, newArgs, 0, args.length - 1);
+    
     if (args[0].equals("symmetric")) {
-      String[] newArgs = new String[args.length - 1];
-      System.arraycopy(args, 1, newArgs, 0, args.length - 1);
       SymmetricMatrixGen.main(newArgs);
+    } else if(args[0].equals("square")) {
+      System.out.println("Not implemented yet.");
+      //SquareMatrixGen.main(newArgs);
     }
   }
 }

Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java
(added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java
Wed Dec 12 12:09:48 2012
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents sparse vector. It will give improvement in memory
+ * consumption in case of vectors which sparsity is close to zero. Can be used
+ * in SpMV for representing input matrix rows efficiently. Internally
+ * represents values as list of indeces and list of values.
+ */
+public class SparseVectorWritable implements Writable {
+
+  private Integer size;
+  private List<Integer> indeces;
+  private List<Double> values;
+
+  public SparseVectorWritable() {
+    indeces = new ArrayList<Integer>();
+    values = new ArrayList<Double>();
+  }
+  
+  public void clear(){
+    indeces = new ArrayList<Integer>();
+    values = new ArrayList<Double>();    
+  }
+
+  public void addCell(int index, double value) {
+    indeces.add(index);
+    values.add(value);
+  }
+
+  public void setSize(int size) {
+    this.size = size;
+  }
+
+  public int getSize() {
+    if (size != null)
+      return size;
+    return indeces.size();
+  }
+
+  public List<Integer> getIndeces() {
+    return indeces;
+  }
+
+  public List<Double> getValues() {
+    return values;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    clear();
+    int size = in.readInt();
+    int len = in.readInt();
+    setSize(size);
+    for (int i = 0; i < len; i++) {
+      int index = in.readInt();
+      double value = in.readDouble();
+      this.addCell(index, value);
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(getSize());
+    out.writeInt(indeces.size());
+    for (int i = 0; i < indeces.size(); i++) {
+      out.writeInt(indeces.get(i));
+      out.writeDouble(values.get(i));
+    }
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder st = new StringBuilder();
+    st.append(" "+getSize()+" "+indeces.size());
+    for (int i = 0; i < indeces.size(); i++) 
+      st.append(" "+indeces.get(i)+" "+values.get(i));
+    return st.toString();
+  }
+
+}
\ No newline at end of file

Added: hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java (added)
+++ hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java Wed Dec 12 12:09:48
2012
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.examples.util.DenseVectorWritable;
+import org.apache.hama.examples.util.SparseVectorWritable;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This class is test cases for {@link SpMV}. It will contain simple hand
+ * calculated cases, and cases of different matrix and vector sizes given with
+ * help of {@link RandomMatrixGenerator}
+ */
+public class SpMVTest {
+
+  protected static final Log LOG = LogFactory.getLog(SpMVTest.class);
+
+  private HamaConfiguration conf;
+  private FileSystem fs;
+  private String baseDir;
+
+  @Before
+  public void prepare() throws IOException {
+    conf = new HamaConfiguration();
+    fs = FileSystem.get(conf);
+    baseDir = fs.getHomeDirectory().toString() + "/spmv";
+  }
+
+  /**
+   * Simple test of running spmv from {@link ExampleDriver}. You should specify
+   * paths.
+   */
+  @Test
+  public void runFromDriver() {
+    try {
+      String matrixPath = "";
+      String vectorPath = "";
+      String outputPath = "";
+      if (matrixPath.isEmpty() || vectorPath.isEmpty() || outputPath.isEmpty()) {
+        LOG.info("Please setup input path for vector and matrix and output path for result.");
+        return;
+      }
+      ExampleDriver.main(new String[] { "spmv", matrixPath, vectorPath,
+          outputPath, "4" });
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getLocalizedMessage());
+    }
+  }
+
+  /**
+   * Simple test. multiplying [1 0 6 0] [2] [38] [0 4 0 0] * [3] = [12] [0 2 3
+   * 0] [6] [24] [3 0 0 5] [1] [11]
+   */
+  @Test
+  public void simpleSpMVTest() {
+    try {
+      HamaConfiguration conf = new HamaConfiguration();
+      String testDir = "/simple/";
+      int size = 4;
+
+      // creating test matrix
+      HashMap<Integer, Writable> inputMatrix = new HashMap<Integer, Writable>();
+      SparseVectorWritable vector0 = new SparseVectorWritable();
+      vector0.setSize(size);
+      vector0.addCell(0, 1);
+      vector0.addCell(2, 6);
+      SparseVectorWritable vector1 = new SparseVectorWritable();
+      vector1.setSize(size);
+      vector1.addCell(1, 4);
+      SparseVectorWritable vector2 = new SparseVectorWritable();
+      vector2.setSize(size);
+      vector2.addCell(1, 2);
+      vector2.addCell(2, 3);
+      SparseVectorWritable vector3 = new SparseVectorWritable();
+      vector3.setSize(size);
+      vector3.addCell(0, 3);
+      vector3.addCell(3, 5);
+      inputMatrix.put(0, vector0);
+      inputMatrix.put(1, vector1);
+      inputMatrix.put(2, vector2);
+      inputMatrix.put(3, vector3);
+      String matrixPath = baseDir + testDir + "inputMatrix";
+      writeMatrix(matrixPath, conf, inputMatrix);
+
+      HashMap<Integer, Writable> inputVector = new HashMap<Integer, Writable>();
+      DenseVectorWritable vector = new DenseVectorWritable();
+      vector.setSize(size);
+      vector.addCell(0, 2);
+      vector.addCell(1, 3);
+      vector.addCell(2, 6);
+      vector.addCell(3, 1);
+      inputVector.put(0, vector);
+      String vectorPath = baseDir + testDir + "inputVector";
+      writeMatrix(vectorPath, conf, inputVector);
+
+      String outputPath = baseDir + testDir;
+      SpMV.main(new String[] { matrixPath, vectorPath, outputPath, "4" });
+
+      String resultPath = SpMV.getResultPath();
+      DenseVectorWritable result = new DenseVectorWritable();
+      SpMV.readFromFile(resultPath, result, conf);
+
+      double expected[] = { 38, 12, 24, 11 };
+      if (result.getSize() != size)
+        throw new Exception("Incorrect size of output vector");
+      for (int i = 0; i < result.getSize(); i++)
+        if ((result.get(i) - expected[i]) < 0.01)
+          expected[i] = 0;
+
+      for (int i = 0; i < expected.length; i++)
+        if (expected[i] != 0)
+          throw new Exception("Result doesn't meets expectations");
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getLocalizedMessage());
+    }
+  }
+
+  /**
+   * Simple test of spmv work with files. You should specify paths.
+   */
+  @Test
+  public void simpleSpMVTestFile() {
+    try {
+      int size = 4;
+      String matrixPath = "";
+      String vectorPath = "";
+      String outputPath = "";
+      if (matrixPath.isEmpty() || vectorPath.isEmpty() || outputPath.isEmpty()) {
+        LOG.info("Please setup input path for vector and matrix and output path for result,
"
+            + "if you want to run this example");
+        return;
+      }
+      SpMV.main(new String[] { matrixPath, vectorPath, outputPath, "4" });
+
+      String resultPath = SpMV.getResultPath();
+      DenseVectorWritable result = new DenseVectorWritable();
+      SpMV.readFromFile(resultPath, result, conf);
+
+      double expected[] = { 38, 12, 24, 11 };
+      if (result.getSize() != size)
+        throw new Exception("Incorrect size of output vector");
+      for (int i = 0; i < result.getSize(); i++)
+        if ((result.get(i) - expected[i]) < 0.01)
+          expected[i] = 0;
+
+      for (int i = 0; i < expected.length; i++)
+        if (expected[i] != 0)
+          throw new Exception("Result doesn't meets expectations");
+
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail(e.getLocalizedMessage());
+    }
+  }
+
+  /**
+   * This method gives the ability to write matrix from memory to file. It
+   * should be used with small matrices and for test purposes only.
+   * 
+   * @param pathString path to file where matrix will be writed.
+   * @param conf configuration
+   * @param matrix map of row indeces and values presented as Writable
+   * @throws IOException
+   */
+  public static void writeMatrix(String pathString, Configuration conf,
+      Map<Integer, Writable> matrix) throws IOException {
+    boolean inited = false;
+    FileSystem fs = FileSystem.get(conf);
+    SequenceFile.Writer writer = null;
+    try {
+      for (Integer index : matrix.keySet()) {
+        IntWritable key = new IntWritable(index);
+        Writable value = matrix.get(index);
+        if (!inited) {
+          writer = new SequenceFile.Writer(fs, conf, new Path(pathString),
+              IntWritable.class, value.getClass());
+          inited = true;
+        }
+        writer.append(key, value);
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    } finally {
+      if (writer != null)
+        writer.close();
+    }
+
+  }
+}



Mime
View raw message