tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject [18/50] [abbrv] tez git commit: TEZ-1735. Allow setting basic info per DAG for Tez UI. (hitesh)
Date Mon, 17 Nov 2014 19:22:27 GMT
TEZ-1735. Allow setting basic info per DAG for Tez UI. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/db4112cc
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/db4112cc
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/db4112cc

Branch: refs/heads/TEZ-8
Commit: db4112cc0019b36c587503ad5673b4a7265cc91a
Parents: 5782716
Author: Hitesh Shah <hitesh@apache.org>
Authored: Mon Nov 3 19:55:57 2014 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Mon Nov 3 19:55:57 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../main/java/org/apache/tez/dag/api/DAG.java   | 26 +++++++++++++++++---
 tez-api/src/main/proto/DAGApiRecords.proto      |  1 +
 .../apache/tez/dag/history/utils/DAGUtils.java  |  4 +++
 .../tez/dag/history/utils/TestDAGUtils.java     |  4 +++
 .../examples/TestOrderedWordCount.java          |  1 +
 6 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f741109..6fab873 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -98,6 +98,7 @@ ALL CHANGES:
   TEZ-1726. Build broken against Hadoop-2.6.0 due to change in NodeReport.
   TEZ-1579. MR examples should be setting mapreduce.framework.name to yarn-tez.
   TEZ-1731. OnDiskMerger can end up clobbering files across tasks with LocalDiskFetch enabled.
+  TEZ-1735. Allow setting basic info per DAG for Tez UI.
 
 Release 0.5.1: 2014-10-02
 

http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 34c27bf..acef4da 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -48,7 +48,6 @@ import org.apache.tez.dag.api.EdgeProperty.DataMovementType;
 import org.apache.tez.dag.api.EdgeProperty.DataSourceType;
 import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
 import org.apache.tez.dag.api.VertexGroup.GroupInfo;
-import org.apache.tez.dag.api.TaskLocationHint;
 import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
 import org.apache.tez.dag.api.records.DAGProtos.EdgePlan;
@@ -87,7 +86,8 @@ public class DAG {
   Set<GroupInputEdge> groupInputEdges = Sets.newHashSet();
   private DAGAccessControls dagAccessControls;
   Map<String, LocalResource> commonTaskLocalFiles = Maps.newHashMap();
-  
+  String dagInfo;
+
   private Stack<String> topologicalVertexStack = new Stack<String>();
 
   private DAG(String name) {
@@ -148,7 +148,24 @@ public class DAG {
     this.credentials = credentials;
     return this;
   }
-  
+
+  /**
+   * Set description info for this DAG that can be used for visualization purposes.
+   * @param dagInfo JSON blob as a serialized string.
+   *                Recognized keys by the UI are:
+   *                    "context" - The application context in which this DAG is being used.
+   *                                For example, this could be set to "Hive" or "Pig" if
+   *                                this is being run as part of a Hive or Pig script.
+   *                    "description" - General description on what this DAG is going to
do.
+   *                                In the case of Hive, this could be the SQL query text.
+   * @return {@link DAG}
+   */
+  public synchronized DAG setDAGInfo(String dagInfo) {
+    Preconditions.checkNotNull(dagInfo);
+    this.dagInfo = dagInfo;
+    return this;
+  }
+
   /**
    * Create a group of vertices that share a common output. This can be used to implement

    * unions efficiently.
@@ -589,6 +606,9 @@ public class DAG {
     DAGPlan.Builder dagBuilder = DAGPlan.newBuilder();
 
     dagBuilder.setName(this.name);
+    if (this.dagInfo != null && !this.dagInfo.isEmpty()) {
+      dagBuilder.setDagInfo(this.dagInfo);
+    }
     
     if (!vertexGroups.isEmpty()) {
       for (VertexGroup av : vertexGroups) {

http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-api/src/main/proto/DAGApiRecords.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/DAGApiRecords.proto b/tez-api/src/main/proto/DAGApiRecords.proto
index 177faba..0539405 100644
--- a/tez-api/src/main/proto/DAGApiRecords.proto
+++ b/tez-api/src/main/proto/DAGApiRecords.proto
@@ -164,6 +164,7 @@ message DAGPlan {
   optional bytes credentials_binary = 5;
   repeated PlanVertexGroupInfo vertex_groups = 6;
   repeated PlanLocalResource local_resource = 7;
+  optional string dag_info = 8;
 }
 
 // DAG monitoring messages

http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
index 0bcbcbe..5f259989 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java
@@ -49,6 +49,7 @@ import org.codehaus.jettison.json.JSONObject;
 public class DAGUtils {
 
   public static final String DAG_NAME_KEY = "dagName";
+  public static final String DAG_INFO_KEY = "dagInfo";
   public static final String VERTICES_KEY = "vertices";
   public static final String EDGES_KEY = "edges";
   public static final String VERTEX_GROUPS_KEY = "vertexGroups";
@@ -138,6 +139,9 @@ public class DAGUtils {
     final int version = 1;
     Map<String,Object> dagMap = new LinkedHashMap<String, Object>();
     dagMap.put(DAG_NAME_KEY, dagPlan.getName());
+    if (dagPlan.hasDagInfo()) {
+      dagMap.put(DAG_INFO_KEY, dagPlan.getDagInfo());
+    }
     dagMap.put(VERSION_KEY, version);
     ArrayList<Object> verticesList = new ArrayList<Object>();
     for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) {

http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
index 6f4202a..496a12c 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
@@ -71,6 +71,7 @@ public class TestDAGUtils {
         dummyTaskCount, dummyTaskResource);
 
     DAG dag = DAG.create("testDag");
+    dag.setDAGInfo("dagInfo");
     String groupName1 = "uv12";
     org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
     OutputDescriptor outDesc = OutputDescriptor.create("output.class")
@@ -110,6 +111,9 @@ public class TestDAGUtils {
 
     Map<String, Object> atsMap = DAGUtils.convertDAGPlanToATSMap(dagPlan);
     Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_NAME_KEY));
+    Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY));
+    Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY));
+    Assert.assertEquals("dagInfo", atsMap.get(DAGUtils.DAG_INFO_KEY));
     Assert.assertEquals(dagPlan.getName(), atsMap.get(DAGUtils.DAG_NAME_KEY));
     Assert.assertTrue(atsMap.containsKey("version"));
     Assert.assertEquals(1, atsMap.get("version"));

http://git-wip-us.apache.org/repos/asf/tez/blob/db4112cc/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index e519085..ec6b429 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -270,6 +270,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
     vertices.add(finalReduceVertex);
 
     DAG dag = DAG.create("OrderedWordCount" + dagIndex);
+    dag.setDAGInfo("{ \"context\": \"Tez\", \"description\": \"TestOrderedWordCount Job\"
}");
     for (int i = 0; i < vertices.size(); ++i) {
       dag.addVertex(vertices.get(i));
     }


Mime
View raw message