hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1527791 - in /hive/branches/tez: eclipse-templates/ ivy/ ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/org/apache/hadoop/hive/ql/plan/
Date Mon, 30 Sep 2013 21:36:42 GMT
Author: gunther
Date: Mon Sep 30 21:36:42 2013
New Revision: 1527791

URL: http://svn.apache.org/r1527791
Log:
HIVE-5387: Need to create edge properties for hive on tez (Vikram Dixit K via Gunther Hagleitner)

Modified:
    hive/branches/tez/eclipse-templates/.classpath
    hive/branches/tez/ivy/libraries.properties
    hive/branches/tez/ql/ivy.xml
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
    hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java

Modified: hive/branches/tez/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/tez/eclipse-templates/.classpath?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/eclipse-templates/.classpath (original)
+++ hive/branches/tez/eclipse-templates/.classpath Mon Sep 30 21:36:42 2013
@@ -75,6 +75,7 @@
   <classpathentry kind="lib" path="build/ivy/lib/default/commons-collections-@commons-collections.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/bonecp-@BoneCP.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/commons-pool-@commons-pool.version@.jar"/>
+  <classpathentry kind="lib" path="build/ivy/lib/default/commons-lang3-@commons-lang3.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/slf4j-api-@slf4j-api.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/slf4j-log4j12-@slf4j-log4j12.version@.jar"/>
   <classpathentry kind="lib" path="build/ivy/lib/default/JavaEWAH-@javaewah.version@.jar"/>

Modified: hive/branches/tez/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/ivy/libraries.properties?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ivy/libraries.properties (original)
+++ hive/branches/tez/ivy/libraries.properties Mon Sep 30 21:36:42 2013
@@ -40,6 +40,7 @@ commons-lang.version=2.4
 commons-logging.version=1.0.4
 commons-logging-api.version=1.0.4
 commons-pool.version=1.5.4
+commons-lang3.version=3.1
 derby.version=10.4.2.0
 guava.version=11.0.2
 hbase.version=0.94.6.1

Modified: hive/branches/tez/ql/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/ivy.xml?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/ivy.xml (original)
+++ hive/branches/tez/ql/ivy.xml Mon Sep 30 21:36:42 2013
@@ -66,7 +66,8 @@
     <dependency org="org.json" name="json" rev="${json.version}"/>
     <dependency org="commons-collections" name="commons-collections" rev="${commons-collections.version}"/>
     <dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
-                transitive="false"/>
+      transitive="false"/>
+    <dependency org="org.apache.commons" name="commons-lang3" rev="${commons-lang3.version}"/>
     <dependency org="com.googlecode.javaewah" name="JavaEWAH" rev="${javaewah.version}"/>
     <dependency org="javolution" name="javolution" rev="${javolution.version}"/>
 

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Mon Sep
30 21:36:42 2013
@@ -174,10 +174,8 @@ public class TezTask extends Task<TezWor
       for (BaseWork v: work.getChildren(w)) {
         assert workToVertex.containsKey(v);
         Edge e = null;
-        EdgeType edgeType = EdgeType.SIMPLE_EDGE;
-        if (work.isBroadCastEdge(w, v)) {
-          edgeType = EdgeType.BROADCAST_EDGE;
-        }
+
+        EdgeType edgeType = work.getEdgeProperty(w, v);
 
         e = DagUtils.createEdge(wxConf, wx, workToConf.get(v), workToVertex.get(v), edgeType);
         dag.addEdge(e);

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java Mon Sep 30 21:36:42
2013
@@ -25,6 +25,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -49,7 +51,8 @@ public class TezWork extends AbstractOpe
   private final Set<BaseWork> leaves = new HashSet<BaseWork>();
   private final Map<BaseWork, List<BaseWork>> workGraph = new HashMap<BaseWork,
List<BaseWork>>();
   private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new HashMap<BaseWork,
List<BaseWork>>();
-  private final Map<BaseWork, List<BaseWork>> broadcastEdge = new HashMap<BaseWork,
List<BaseWork>>();
+  private final Map<Pair<BaseWork, BaseWork>, EdgeType> edgeProperties =
+      new HashMap<Pair<BaseWork, BaseWork>, EdgeType>();
 
   /**
    * getAllWork returns a topologically sorted list of BaseWork
@@ -95,7 +98,6 @@ public class TezWork extends AbstractOpe
     }
     workGraph.put(w, new LinkedList<BaseWork>());
     invertedWorkGraph.put(w, new LinkedList<BaseWork>());
-    broadcastEdge.put(w, new LinkedList<BaseWork>());
     roots.add(w);
     leaves.add(w);
   }
@@ -109,13 +111,8 @@ public class TezWork extends AbstractOpe
     invertedWorkGraph.get(b).add(a);
     roots.remove(b);
     leaves.remove(a);
-    switch (edgeType) {
-    case BROADCAST_EDGE:
-      broadcastEdge.get(a).add(b);
-      break;
-    default:
-      break;
-    }
+    ImmutablePair workPair = new ImmutablePair(a, b);
+    edgeProperties.put(workPair, edgeType);
   }
 
   /**
@@ -201,11 +198,7 @@ public class TezWork extends AbstractOpe
     invertedWorkGraph.remove(work);
   }
 
-  // checks if a and b need a broadcast edge between them
-  public boolean isBroadCastEdge(BaseWork a, BaseWork b) {
-    if ((broadcastEdge.get(a).contains(b)) || (broadcastEdge.get(b).contains(a))) {
-      return true;
-    }
-    return false;
+  public EdgeType getEdgeProperty(BaseWork a, BaseWork b) {
+    return edgeProperties.get(new ImmutablePair(a,b));
   }
 }

Modified: hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java (original)
+++ hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java Mon Sep
30 21:36:42 2013
@@ -77,6 +77,8 @@ public class TestTezWork {
       Assert.assertEquals(work.getParents(w).size(), 0);
       Assert.assertEquals(work.getChildren(w).size(), 0);
     }
+
+    Assert.assertEquals(work.getEdgeProperty(parent, child), EdgeType.SIMPLE_EDGE);
   }
 
   @Test
@@ -99,6 +101,8 @@ public class TestTezWork {
       Assert.assertEquals(work.getParents(w).size(), 0);
       Assert.assertEquals(work.getChildren(w).size(), 0);
     }
+
+    Assert.assertEquals(work.getEdgeProperty(parent, child), EdgeType.BROADCAST_EDGE);
   }
 
   @Test



Mime
View raw message