tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject svn commit: r1477252 - in /incubator/tez/branches/TEZ-1: tez-dag-api/src/main/java/org/apache/tez/dag/api/ tez-dag/src/main/java/org/apache/tez/dag/app/ tez-yarn-client/src/main/java/org/apache/tez/mapreduce/
Date Mon, 29 Apr 2013 18:27:35 GMT
Author: hitesh
Date: Mon Apr 29 18:27:34 2013
New Revision: 1477252

URL: http://svn.apache.org/r1477252
Log:
TEZ-28. Remove default constructor for EdgeProperty. (hitesh)

Modified:
    incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
    incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
    incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
    incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java

Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
Mon Apr 29 18:27:34 2013
@@ -235,7 +235,8 @@ public class DAGConfiguration extends Co
     for(Map.Entry<String,LocalResource> entry : lrs.entrySet()) {
       LocalResource lr = entry.getValue();
       try {
-        String lrStr = StringUtils.escapeString(entry.getKey(), StringUtils.ESCAPE_CHAR,
+        String lrStr = StringUtils.escapeString(entry.getKey(),
+            StringUtils.ESCAPE_CHAR,
             SEPARATOR.charAt(0))
             + SEPARATOR
             + StringUtils.escapeString(
@@ -243,23 +244,28 @@ public class DAGConfiguration extends Co
                 StringUtils.ESCAPE_CHAR,
                 SEPARATOR.charAt(0))
             + SEPARATOR
-            + StringUtils.escapeString(String.valueOf(lr.getSize()), StringUtils.ESCAPE_CHAR,
SEPARATOR.charAt(0))
+            + StringUtils.escapeString(String.valueOf(lr.getSize()),
+                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
             + SEPARATOR
-            + StringUtils.escapeString(String.valueOf(lr.getTimestamp()), StringUtils.ESCAPE_CHAR,
SEPARATOR.charAt(0))
+            + StringUtils.escapeString(String.valueOf(lr.getTimestamp()),
+                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
             + SEPARATOR
-            + StringUtils.escapeString(lr.getType().name(), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+            + StringUtils.escapeString(lr.getType().name(),
+                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
             + SEPARATOR
-            + StringUtils.escapeString(lr.getVisibility().name(), StringUtils.ESCAPE_CHAR,
SEPARATOR.charAt(0))
+            + StringUtils.escapeString(lr.getVisibility().name(),
+                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
             + SEPARATOR
-            + StringUtils.escapeString((lr.getPattern() == null ? "" : lr.getPattern()),
StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
-        LOG.info("XXXX serializing local resource:"
-            + lrStr);
+            + StringUtils.escapeString(
+                (lr.getPattern() == null ? "" : lr.getPattern()),
+                StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
         lrStrs[i++] = StringUtils.escapeString(lrStr);
       } catch (URISyntaxException e) {
         throw new RuntimeException(e);
       }
     }
-    setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(), lrStrs);
+    setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(),
+        lrStrs);
   }
 
   public Map<String, LocalResource> getVertexLocalResources(String vertexName) {
@@ -272,7 +278,6 @@ public class DAGConfiguration extends Co
     }
     LOG.info("XXXX Found " + lrStrs.length + " local resources");
     for (String lrStr : lrStrs) {
-      LOG.info("XXXX Parsing local resource from " + lrStr);
       String[] tokens =
           StringUtils.split(
               lrStr, StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
@@ -299,7 +304,8 @@ public class DAGConfiguration extends Co
             + ", vertexName=" + vertexName
             + ", resourceName=" + resourceName
             + ", resourceUrl"
-            + ConverterUtils.getPathFromYarnURL(lRsrc.getResource()).toString());
+            + ConverterUtils.getPathFromYarnURL(
+                lRsrc.getResource()).toString());
       } catch (URISyntaxException e) {
         // Ignore
         // FIXME

Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
Mon Apr 29 18:27:34 2013
@@ -46,14 +46,6 @@ public class EdgeProperty { // FIXME ren
     this.outputClass = outputClass;
   }
   
-  // default constructor for mapreduce case
-  public EdgeProperty() {
-    this.connectionPattern = ConnectionPattern.BIPARTITE;
-    this.sourceType = SourceType.STABLE;
-    this.inputClass = "org.apache.tez.engine.lib.input.ShuffledMergedInput";
-    this.outputClass = "org.apache.tez.engine.lib.output.OnFileSortedOutput";
-  }
-  
   public ConnectionPattern getConnectionPattern() {
     return connectionPattern;
   }

Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
Mon Apr 29 18:27:34 2013
@@ -20,8 +20,12 @@ import org.apache.hadoop.yarn.util.Build
 import org.apache.tez.dag.api.DAGConfiguration;
 import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
+import org.apache.tez.engine.lib.input.ShuffledMergedInput;
+import org.apache.tez.engine.lib.output.OnFileSortedOutput;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 public class MRRExampleHelper {
@@ -88,8 +92,16 @@ public class MRRExampleHelper {
        "org.apache.tez.mapreduce.task.IntermediateTask", 3);
    Vertex reduce2Vertex = new Vertex("reduce2",
        "org.apache.tez.mapreduce.task.FinalTask", 3);
-   Edge edge1 = new Edge(mapVertex, reduce1Vertex, new EdgeProperty());
-   Edge edge2 = new Edge(reduce1Vertex, reduce2Vertex, new EdgeProperty());
+   Edge edge1 = new Edge(mapVertex, reduce1Vertex,
+       new EdgeProperty(ConnectionPattern.BIPARTITE,
+           SourceType.STABLE,
+           ShuffledMergedInput.class.getName(),
+           OnFileSortedOutput.class.getName()));
+   Edge edge2 = new Edge(reduce1Vertex, reduce2Vertex,
+       new EdgeProperty(ConnectionPattern.BIPARTITE,
+           SourceType.STABLE,
+           ShuffledMergedInput.class.getName(),
+           OnFileSortedOutput.class.getName()));
    Map<String, LocalResource> jobRsrcs = createLocalResources(getMRRBaseDir(),
        getMRRLocalRsrcList());
 
@@ -146,7 +158,11 @@ public class MRRExampleHelper {
        "org.apache.tez.mapreduce.task.InitialTask", 6);
    Vertex reduceVertex = new Vertex("reduce",
        "org.apache.tez.mapreduce.task.FinalTask", 1);
-   Edge edge = new Edge(mapVertex, reduceVertex, new EdgeProperty());
+   Edge edge = new Edge(mapVertex, reduceVertex,
+       new EdgeProperty(ConnectionPattern.BIPARTITE,
+           SourceType.STABLE,
+           ShuffledMergedInput.class.getName(),
+           OnFileSortedOutput.class.getName()));
 
    Map<String, LocalResource> jobRsrcs = createLocalResources(getMRBaseDir(),
        getMRLocalRsrcList());

Modified: incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
(original)
+++ incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
Mon Apr 29 18:27:34 2013
@@ -59,7 +59,11 @@ import org.apache.tez.dag.api.Edge;
 import org.apache.tez.dag.api.EdgeProperty;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.engine.lib.input.ShuffledMergedInput;
+import org.apache.tez.engine.lib.output.OnFileSortedOutput;
 import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 import org.apache.tez.mapreduce.hadoop.MultiStageMRConfToTezTranslator;
@@ -645,7 +649,11 @@ public class YARNRunner implements Clien
           + ", parrellism=" + reduceVertex.getParallelism());
       dag.addVertex(reduceVertex);
 
-      EdgeProperty edgeProperty = new EdgeProperty();
+      EdgeProperty edgeProperty =
+          new EdgeProperty(ConnectionPattern.BIPARTITE,
+              SourceType.STABLE,
+              ShuffledMergedInput.class.getName(),
+              OnFileSortedOutput.class.getName());
       Edge edge = null;
       if (!isMRR) {
         edge = new Edge(mapVertex, reduceVertex, edgeProperty);



Mime
View raw message