hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r1437860 - in /hama/trunk: ./ core/ examples/src/main/java/org/apache/hama/examples/ examples/src/main/java/org/apache/hama/examples/util/ examples/src/test/java/org/apache/hama/examples/
Date Thu, 24 Jan 2013 05:20:39 GMT
Author: edwardyoon
Date: Thu Jan 24 05:20:38 2013
New Revision: 1437860

URL: http://svn.apache.org/viewvc?rev=1437860&view=rev
Log:
HAMA-714: Align format consistency between examples and generators

Removed:
    hama/trunk/examples/src/main/java/org/apache/hama/examples/util/VertexInputGen.java
Modified:
    hama/trunk/CHANGES.txt
    hama/trunk/core/pom.xml
    hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java
    hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java
    hama/trunk/pom.xml

Modified: hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hama/trunk/CHANGES.txt?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/CHANGES.txt (original)
+++ hama/trunk/CHANGES.txt Thu Jan 24 05:20:38 2013
@@ -16,6 +16,7 @@ Release 0.7 (unreleased changes)
 
   IMPROVEMENTS
 
+   HAMA-714: Align format consistency between examples and generators (edwardyoon)
    HAMA-531: Reimplementation of partitioner (edwardyoon)
 
 Release 0.6 - November 28, 2012

Modified: hama/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/core/pom.xml?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/core/pom.xml (original)
+++ hama/trunk/core/pom.xml Thu Jan 24 05:20:38 2013
@@ -109,18 +109,6 @@
       <artifactId>hadoop-test</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.avro</groupId>
-      <artifactId>avro-ipc</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.jboss.netty</groupId>
-      <artifactId>netty</artifactId>
-    </dependency>
-    <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-core-asl</artifactId>
     </dependency>

Modified: hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java (original)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/PageRank.java Thu Jan 24 05:20:38
2013
@@ -25,12 +25,13 @@ import java.util.Iterator;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
 import org.apache.hama.HamaConfiguration;
 import org.apache.hama.bsp.HashPartitioner;
 import org.apache.hama.bsp.SequenceFileInputFormat;
+import org.apache.hama.bsp.TextArrayWritable;
 import org.apache.hama.bsp.TextOutputFormat;
 import org.apache.hama.graph.AbstractAggregator;
 import org.apache.hama.graph.AverageAggregator;
@@ -44,34 +45,6 @@ import org.apache.hama.graph.VertexInput
  */
 public class PageRank {
 
-  public static class PagerankTextReader extends
-      VertexInputReader<LongWritable, Text, Text, NullWritable, DoubleWritable> {
-
-    /**
-     * The text file essentially should look like: <br/>
-     * VERTEX_ID\t(n-tab separated VERTEX_IDs)<br/>
-     * E.G:<br/>
-     * 1\t2\t3\t4<br/>
-     * 2\t3\t1<br/>
-     * etc.
-     */
-    @Override
-    public boolean parseVertex(LongWritable key, Text value,
-        Vertex<Text, NullWritable, DoubleWritable> vertex) throws Exception {
-      String[] split = value.toString().split("\t");
-      for (int i = 0; i < split.length; i++) {
-        if (i == 0) {
-          vertex.setVertexID(new Text(split[i]));
-        } else {
-          vertex
-              .addEdge(new Edge<Text, NullWritable>(new Text(split[i]), null));
-        }
-      }
-      return true;
-    }
-
-  }
-
   public static class PageRankVertex extends
       Vertex<Text, NullWritable, DoubleWritable> {
 
@@ -153,24 +126,44 @@ public class PageRank {
 
   }
 
-  private static void printUsage() {
-    System.out.println("Usage: <input> <output> [tasks]");
-    System.exit(-1);
+  public static class DanglingNodeAggregator
+      extends
+      AbstractAggregator<DoubleWritable, Vertex<Text, NullWritable, DoubleWritable>>
{
+
+    double danglingNodeSum;
+
+    @Override
+    public void aggregate(Vertex<Text, NullWritable, DoubleWritable> vertex,
+        DoubleWritable value) {
+      if (vertex != null) {
+        if (vertex.getEdges().size() == 0) {
+          danglingNodeSum += value.get();
+        }
+      } else {
+        danglingNodeSum += value.get();
+      }
+    }
+
+    @Override
+    public DoubleWritable getValue() {
+      return new DoubleWritable(danglingNodeSum);
+    }
+
   }
 
-  public static void main(String[] args) throws IOException,
-      InterruptedException, ClassNotFoundException {
-    if (args.length < 2)
-      printUsage();
+  public static class PagerankSeqReader
+      extends
+      VertexInputReader<Text, TextArrayWritable, Text, NullWritable, DoubleWritable>
{
+    @Override
+    public boolean parseVertex(Text key, TextArrayWritable value,
+        Vertex<Text, NullWritable, DoubleWritable> vertex) throws Exception {
+      vertex.setVertexID(key);
 
-    HamaConfiguration conf = new HamaConfiguration(new Configuration());
-    GraphJob pageJob = createJob(args, conf);
-    pageJob.setVertexInputReaderClass(PagerankTextReader.class);
+      for (Writable v : value.get()) {
+        vertex.addEdge(new Edge<Text, NullWritable>((Text) v, null));
+      }
 
-    long startTime = System.currentTimeMillis();
-    if (pageJob.waitForCompletion(true)) {
-      System.out.println("Job Finished in "
-          + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
+      return true;
     }
   }
 
@@ -179,7 +172,7 @@ public class PageRank {
       throws IOException {
     GraphJob pageJob = new GraphJob(conf, PageRank.class);
     pageJob.setJobName("Pagerank");
-
+    
     pageJob.setVertexClass(PageRankVertex.class);
     pageJob.setInputPath(new Path(args[0]));
     pageJob.setOutputPath(new Path(args[1]));
@@ -197,6 +190,9 @@ public class PageRank {
     pageJob.setAggregatorClass(AverageAggregator.class,
         DanglingNodeAggregator.class);
 
+    // Vertex reader
+    pageJob.setVertexInputReaderClass(PagerankSeqReader.class);
+    
     pageJob.setVertexIDClass(Text.class);
     pageJob.setVertexValueClass(DoubleWritable.class);
     pageJob.setEdgeValueClass(NullWritable.class);
@@ -210,28 +206,23 @@ public class PageRank {
     return pageJob;
   }
 
-  public static class DanglingNodeAggregator
-      extends
-      AbstractAggregator<DoubleWritable, Vertex<Text, NullWritable, DoubleWritable>>
{
+  private static void printUsage() {
+    System.out.println("Usage: <input> <output> [tasks]");
+    System.exit(-1);
+  }
 
-    double danglingNodeSum;
+  public static void main(String[] args) throws IOException,
+      InterruptedException, ClassNotFoundException {
+    if (args.length < 2)
+      printUsage();
 
-    @Override
-    public void aggregate(Vertex<Text, NullWritable, DoubleWritable> vertex,
-        DoubleWritable value) {
-      if (vertex != null) {
-        if (vertex.getEdges().size() == 0) {
-          danglingNodeSum += value.get();
-        }
-      } else {
-        danglingNodeSum += value.get();
-      }
-    }
+    HamaConfiguration conf = new HamaConfiguration(new Configuration());
+    GraphJob pageJob = createJob(args, conf);
 
-    @Override
-    public DoubleWritable getValue() {
-      return new DoubleWritable(danglingNodeSum);
+    long startTime = System.currentTimeMillis();
+    if (pageJob.waitForCompletion(true)) {
+      System.out.println("Job Finished in "
+          + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
     }
-
   }
 }

Modified: hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java (original)
+++ hama/trunk/examples/src/test/java/org/apache/hama/examples/PageRankTest.java Thu Jan 24
05:20:38 2013
@@ -28,8 +28,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hama.HamaConfiguration;
-import org.apache.hama.examples.PageRank.PageRankVertex;
-import org.apache.hama.examples.util.VertexInputGen;
+import org.apache.hama.examples.util.SymmetricMatrixGen;
 import org.apache.hama.graph.GraphJob;
 import org.apache.hama.graph.GraphJobRunner;
 
@@ -87,11 +86,7 @@ public class PageRankTest extends TestCa
 
   private void generateTestData() throws ClassNotFoundException,
       InterruptedException, IOException {
-    HamaConfiguration conf = new HamaConfiguration();
-    conf.setInt(VertexInputGen.SIZE_OF_MATRIX, 40);
-    conf.setInt(VertexInputGen.DENSITY, 10);
-    conf.setInt("hama.test.vertexcreatorid", 1);
-    VertexInputGen.runJob(conf, 3, INPUT, PageRankVertex.class);
+    SymmetricMatrixGen.main(new String[] { "40", "10", INPUT, "2" });
   }
 
   private void deleteTempDirs() {

Modified: hama/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1437860&r1=1437859&r2=1437860&view=diff
==============================================================================
--- hama/trunk/pom.xml (original)
+++ hama/trunk/pom.xml Thu Jan 24 05:20:38 2013
@@ -214,21 +214,6 @@
         <version>${zookeeper.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.apache.avro</groupId>
-        <artifactId>avro</artifactId>
-        <version>1.6.0</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.avro</groupId>
-        <artifactId>avro-ipc</artifactId>
-        <version>1.6.0</version>
-      </dependency>
-      <dependency>
-        <groupId>org.jboss.netty</groupId>
-        <artifactId>netty</artifactId>
-        <version>3.2.6.Final</version>
-      </dependency>
-      <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-core-asl</artifactId>
         <version>1.9.2</version>



Mime
View raw message