hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1173525 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-yarn/ hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/ hadoop-yarn/hadoop-y...
Date Wed, 21 Sep 2011 08:40:18 GMT
Author: vinodkv
Date: Wed Sep 21 08:40:17 2011
New Revision: 1173525

URL: http://svn.apache.org/viewvc?rev=1173525&view=rev
Log:
MAPREDUCE-2930. svn merge -c r1173524 --ignore-ancestry ../../trunk/

Added:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Graph.java
      - copied unchanged from r1173524, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Graph.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/VisualizeStateMachine.java
      - copied unchanged from r1173524, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/VisualizeStateMachine.java
Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/README
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1173525&r1=1173524&r2=1173525&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Sep 21 08:40:17
2011
@@ -41,6 +41,9 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-2037. Capture intermediate progress, CPU and memory usage for
     tasks. (Dick King via acmurthy) 
 
+    MAPREDUCE-2930. Added the ability to be able to generate graphs from the
+    state-machine definitions. (Binglin Chang via vinodkv)
+
   IMPROVEMENTS
 
     MAPREDUCE-2187. Reporter sends progress during sort/merge. (Anupam Seth via

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1173525&r1=1173524&r2=1173525&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
Wed Sep 21 08:40:17 2011
@@ -113,4 +113,41 @@
       </plugin>
     </plugins>
   </build>
+
+  <profiles>
+    <profile>
+      <id>visualize</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <version>1.2</version>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>java</goal>
+                </goals>
+                <configuration>
+                  <classpathScope>test</classpathScope>
+                  <mainClass>org.apache.hadoop.yarn.util.VisualizeStateMachine</mainClass>
+                  <arguments>
+                    <argument>MapReduce</argument>
+                    <argument>org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl,
+                      org.apache.hadoop.mapreduce.v2.app.job.impl.TaskImpl,
+                      org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl</argument>
+                    <argument>MapReduce.gv</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/README
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/README?rev=1173525&r1=1173524&r2=1173525&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/README (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/README Wed Sep
21 08:40:17 2011
@@ -30,7 +30,6 @@ clean and test: mvn clean install
 run selected test after compile: mvn test -Dtest=TestClassName (combined: mvn clean install
-Dtest=TestClassName)
 create runnable binaries after install: mvn assembly:assembly (combined: mvn clean install
assembly:assembly)
 
-
 Eclipse Projects
 ----------------
 http://maven.apache.org/guides/mini/guide-ide-eclipse.html
@@ -71,3 +70,16 @@ hadoop-yarn-server - Implementation of t
 	hadoop-yarn-server-common - APIs shared between resourcemanager and nodemanager
 	hadoop-yarn-server-nodemanager (TaskTracker replacement)
 	hadoop-yarn-server-resourcemanager (JobTracker replacement)
+
+Utilities for understanding the code
+------------------------------------
+Almost all of the yarn components as well as the mapreduce framework use
+state-machines for all the data objects. To understand those central pieces of
+the code, a visual representation of the state-machines helps much. You can first
+convert the state-machines into graphviz(.gv) format by
+running:
+   mvn compile -Pvisualize
+Then you can use the dot program for generating directed graphs and convert the above
+.gv files to images. The graphviz package has the needed dot program and related
+utilites.For e.g., to generate png files you can run:
+   dot -Tpng NodeManager.gv > NodeManager.png

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java?rev=1173525&r1=1173524&r2=1173525&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
Wed Sep 21 08:40:17 2011
@@ -20,10 +20,14 @@ package org.apache.hadoop.yarn.state;
 
 import java.util.EnumMap;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.Stack;
 
+import org.apache.hadoop.yarn.util.Graph;
+
 /**
  * State machine topology.
  * This object is semantically immutable.  If you have a
@@ -441,4 +445,39 @@ final public class StateMachineFactory
       return currentState;
     }
   }
+
+  /**
+   * Generate a graph represents the state graph of this StateMachine
+   * @param name graph name
+   * @return Graph object generated
+   */
+  public Graph generateStateGraph(String name) {
+    maybeMakeStateMachineTable();
+    Graph g = new Graph(name);
+    for (STATE startState : stateMachineTable.keySet()) {
+      Map<EVENTTYPE, Transition<OPERAND, STATE, EVENTTYPE, EVENT>> transitions
+          = stateMachineTable.get(startState);
+      for (Entry<EVENTTYPE, Transition<OPERAND, STATE, EVENTTYPE, EVENT>> entry
:
+         transitions.entrySet()) {
+        Transition<OPERAND, STATE, EVENTTYPE, EVENT> transition = entry.getValue();
+        if (transition instanceof StateMachineFactory.SingleInternalArc) {
+          StateMachineFactory.SingleInternalArc sa
+              = (StateMachineFactory.SingleInternalArc) transition;
+          Graph.Node fromNode = g.getNode(startState.toString());
+          Graph.Node toNode = g.getNode(sa.postState.toString());
+          fromNode.addEdge(toNode, entry.getKey().toString());
+        } else if (transition instanceof StateMachineFactory.MultipleInternalArc) {
+          StateMachineFactory.MultipleInternalArc ma
+              = (StateMachineFactory.MultipleInternalArc) transition;
+          Iterator<STATE> iter = ma.validPostStates.iterator();
+          while (iter.hasNext()) {
+            Graph.Node fromNode = g.getNode(startState.toString());
+            Graph.Node toNode = g.getNode(iter.next().toString());
+            fromNode.addEdge(toNode, entry.getKey().toString());
+          }
+        }
+      }
+    }
+    return g;
+  }
 }

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1173525&r1=1173524&r2=1173525&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
Wed Sep 21 08:40:17 2011
@@ -103,6 +103,39 @@
         <activeByDefault>true</activeByDefault>
       </activation>
     </profile>
+    <profile>
+      <id>visualize</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <version>1.2</version>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>java</goal>
+                </goals>
+                <configuration>
+                  <mainClass>org.apache.hadoop.yarn.util.VisualizeStateMachine</mainClass>
+                  <arguments>
+                    <argument>NodeManager</argument>
+                    <argument>org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl,
+                       org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl,
+                       org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalizedResource</argument>
+                    <argument>NodeManager.gv</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
   <build>

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1173525&r1=1173524&r2=1173525&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
Wed Sep 21 08:40:17 2011
@@ -98,4 +98,41 @@
       </plugin>
     </plugins>
   </build>
+
+  <profiles>
+    <profile>
+      <id>visualize</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <version>1.2</version>
+            <executions>
+              <execution>
+                <phase>compile</phase>
+                <goals>
+                  <goal>java</goal>
+                </goals>
+                <configuration>
+                  <mainClass>org.apache.hadoop.yarn.util.VisualizeStateMachine</mainClass>
+                  <arguments>
+                    <argument>ResourceManager</argument>
+                    <argument>org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl,
+                      org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl,
+                      org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl,
+                      org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl</argument>
+                    <argument>ResourceManager.gv</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 </project>



Mime
View raw message