hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r1354358 - in /hama/trunk: core/pom.xml dist/pom.xml examples/pom.xml graph/pom.xml graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java ml/pom.xml pom.xml yarn/pom.xml
Date Wed, 27 Jun 2012 08:03:35 GMT
Author: edwardyoon
Date: Wed Jun 27 08:03:33 2012
New Revision: 1354358

URL: http://svn.apache.org/viewvc?rev=1354358&view=rev
Log:
To avoid a lot of data exchanges among peers

Modified:
    hama/trunk/core/pom.xml
    hama/trunk/dist/pom.xml
    hama/trunk/examples/pom.xml
    hama/trunk/graph/pom.xml
    hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
    hama/trunk/ml/pom.xml
    hama/trunk/pom.xml
    hama/trunk/yarn/pom.xml

Modified: hama/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/core/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/core/pom.xml (original)
+++ hama/trunk/core/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
   <parent>
     <groupId>org.apache.hama</groupId>
     <artifactId>hama-parent</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.hama</groupId>
   <artifactId>hama-core</artifactId>
   <name>core</name>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>

Modified: hama/trunk/dist/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/dist/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/dist/pom.xml (original)
+++ hama/trunk/dist/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.hama</groupId>
     <artifactId>hama-parent</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
   
   <artifactId>hama-dist</artifactId>

Modified: hama/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/examples/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/examples/pom.xml (original)
+++ hama/trunk/examples/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
   <parent>
     <groupId>org.apache.hama</groupId>
     <artifactId>hama-parent</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.hama</groupId>
   <artifactId>hama-examples</artifactId>
   <name>examples</name>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>

Modified: hama/trunk/graph/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/graph/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/graph/pom.xml (original)
+++ hama/trunk/graph/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
   <parent>
     <groupId>org.apache.hama</groupId>
     <artifactId>hama-parent</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.hama</groupId>
   <artifactId>hama-graph</artifactId>
   <name>graph</name>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>

Modified: hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
URL: http://svn.apache.org/viewvc/hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java (original)
+++ hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java Wed Jun 27 08:03:33
2012
@@ -391,16 +391,16 @@ public final class GraphJobRunner<V exte
     Vertex<V, E, M> vertex = newVertexInstance(vertexClass, conf);
     vertex.setPeer(peer);
     vertex.runner = this;
-    while (true) {
-      KeyValuePair<Writable, Writable> next = peer.readNext();
-      if (next == null) {
-        break;
-      }
+    
+    KeyValuePair<Writable, Writable> next = null;
+    int lines = 0;
+    while ((next = peer.readNext()) != null) {
       boolean vertexFinished = reader.parseVertex(next.getKey(),
           next.getValue(), vertex);
       if (!vertexFinished) {
         continue;
       }
+      
       if (vertex.getEdges() == null) {
         vertex.setEdges(new ArrayList<Edge<V, E>>(0));
       }
@@ -420,12 +420,26 @@ public final class GraphJobRunner<V exte
         }
         peer.send(peer.getPeerName(partition), new GraphJobMessage(vertex));
       } else {
+        // FIXME need to set destination names
         vertex.setup(conf);
         vertices.put(vertex.getVertexID(), vertex);
       }
       vertex = newVertexInstance(vertexClass, conf);
       vertex.setPeer(peer);
       vertex.runner = this;
+      
+      lines++;
+      if((lines % 50000) == 0) {
+        peer.sync();
+        GraphJobMessage msg = null;
+        while ((msg = peer.getCurrentMessage()) != null) {
+          Vertex<V, E, M> messagedVertex = (Vertex<V, E, M>) msg.getVertex();
+          messagedVertex.setPeer(peer);
+          messagedVertex.runner = this;
+          messagedVertex.setup(conf);
+          vertices.put(messagedVertex.getVertexID(), messagedVertex);
+        }
+      }
     }
 
     if (runtimePartitioning) {
@@ -440,6 +454,8 @@ public final class GraphJobRunner<V exte
       }
     }
 
+    LOG.info("Loading finished at " + peer.getSuperstepCount() + " steps.");
+    
     /*
      * If the user want to repair the graph, it should traverse through that
      * local chunk of adjancency list and message the corresponding peer to

Modified: hama/trunk/ml/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/ml/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/ml/pom.xml (original)
+++ hama/trunk/ml/pom.xml Wed Jun 27 08:03:33 2012
@@ -20,14 +20,14 @@
   <parent>
     <groupId>org.apache.hama</groupId>
     <artifactId>hama-parent</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.hama</groupId>
   <artifactId>hama-ml</artifactId>
   <name>machine learning</name>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>

Modified: hama/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/pom.xml (original)
+++ hama/trunk/pom.xml Wed Jun 27 08:03:33 2012
@@ -27,7 +27,7 @@
   <groupId>org.apache.hama</groupId>
   <artifactId>hama-parent</artifactId>
   <name>Apache Hama parent POM</name>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.5.0-SNAPSHOT</version>
   <url>http://hama.apache.org</url>
   <packaging>pom</packaging>
   <prerequisites>

Modified: hama/trunk/yarn/pom.xml
URL: http://svn.apache.org/viewvc/hama/trunk/yarn/pom.xml?rev=1354358&r1=1354357&r2=1354358&view=diff
==============================================================================
--- hama/trunk/yarn/pom.xml (original)
+++ hama/trunk/yarn/pom.xml Wed Jun 27 08:03:33 2012
@@ -19,14 +19,14 @@
   <parent>
     <groupId>org.apache.hama</groupId>
     <artifactId>hama-parent</artifactId>
-    <version>0.6.0-SNAPSHOT</version>
+    <version>0.5.0-SNAPSHOT</version>
   </parent>
 
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.hama</groupId>
   <artifactId>hama-yarn</artifactId>
   <name>yarn</name>
-  <version>0.6.0-SNAPSHOT</version>
+  <version>0.5.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <dependencies>



Mime
View raw message