hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1661844 - in /hive/branches/llap: data/conf/tez/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/results/clientpositive/tez/
Date Tue, 24 Feb 2015 06:16:11 GMT
Author: gunther
Date: Tue Feb 24 06:16:11 2015
New Revision: 1661844

URL: http://svn.apache.org/r1661844
Log:
HIVE-9765: LLAP: uber mode where applicable (Gunther Hagleitner)

Modified:
    hive/branches/llap/data/conf/tez/hive-site.xml
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
    hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
    hive/branches/llap/ql/src/test/results/clientpositive/tez/llapdecider.q.out

Modified: hive/branches/llap/data/conf/tez/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/data/conf/tez/hive-site.xml?rev=1661844&r1=1661843&r2=1661844&view=diff
==============================================================================
Binary files - no diff available.

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1661844&r1=1661843&r2=1661844&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue
Feb 24 06:16:11 2015
@@ -1574,9 +1574,16 @@ public final class FunctionRegistry {
    * @return True iff the fnExpr represents a hive built-in function (native, non-permanent)
    */
   public static boolean isBuiltInFuncExpr(ExprNodeGenericFuncDesc fnExpr) {
-    Class<?> udfClass = FunctionRegistry.getGenericUDFClassFromExprDesc(fnExpr);
-    if (udfClass != null) {
-      return system.isBuiltInFunc(udfClass);
+    GenericUDF udf = fnExpr.getGenericUDF();
+    if (udf == null) return false;
+
+    Class clazz = udf.getClass();
+    if (udf instanceof GenericUDFBridge) {
+      clazz = ((GenericUDFBridge)udf).getUdfClass();
+    }
+
+    if (clazz != null) {
+      return system.isBuiltInFunc(clazz);
     }
     return false;
   }
@@ -1590,4 +1597,4 @@ public final class FunctionRegistry {
       String blackListStr) {
     system.setupPermissionsForUDFs(whiteListStr, blackListStr);
   }
-}
\ No newline at end of file
+}

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java?rev=1661844&r1=1661843&r2=1661844&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
(original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
Tue Feb 24 06:16:11 2015
@@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
+import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.Statistics;
 import org.apache.hadoop.hive.ql.plan.TezWork;
 
@@ -131,6 +132,14 @@ public class LlapDecider implements Phys
 
     private void convertWork(TezWork tezWork, BaseWork work)
       throws SemanticException {
+      // let's see if we can go one step further and just uber this puppy
+      if (tezWork.getChildren(work).isEmpty()
+	  && work instanceof ReduceWork
+	  && ((ReduceWork) work).getNumReduceTasks() == 1) {
+	work.setUberMode(true);
+      }
+
+      // always mark as llap
       work.setLlapMode(true);
     }
 

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1661844&r1=1661843&r2=1661844&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Tue Feb 24
06:16:11 2015
@@ -316,12 +316,16 @@ public class MapWork extends BaseWork {
   public String getExecutionMode() {
     if (vectorMode) {
       if (llapMode) {
-	return "vectorized, llap";
+	if (uberMode) {
+	  return "vectorized, uber";
+	} else {
+	  return "vectorized, llap";
+	}
       } else {
 	return "vectorized";
       }
     } else if (llapMode) {
-      return "llap";
+      return uberMode? "uber" : "llap";
     }
     return null;
   }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java?rev=1661844&r1=1661843&r2=1661844&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java Tue Feb
24 06:16:11 2015
@@ -152,12 +152,16 @@ public class ReduceWork extends BaseWork
   public String getExecutionMode() {
     if (vectorMode) {
       if (llapMode) {
-	return "vectorized, llap";
+	if (uberMode) {
+	  return "vectorized, uber";
+	} else {
+	  return "vectorized, llap";
+	}
       } else {
 	return "vectorized";
       }
     } else if (llapMode) {
-      return "llap";
+      return uberMode? "uber" : "llap";
     }
     return null;
   }

Modified: hive/branches/llap/ql/src/test/results/clientpositive/tez/llapdecider.q.out
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/results/clientpositive/tez/llapdecider.q.out?rev=1661844&r1=1661843&r2=1661844&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/results/clientpositive/tez/llapdecider.q.out (original)
+++ hive/branches/llap/ql/src/test/results/clientpositive/tez/llapdecider.q.out Tue Feb 24
06:16:11 2015
@@ -203,7 +203,7 @@ STAGE PLANS:
                   Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column
stats: NONE
                   value expressions: _col0 (type: string)
         Reducer 3 
-            Execution mode: llap
+            Execution mode: uber
             Reduce Operator Tree:
               Select Operator
                 expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
@@ -284,7 +284,7 @@ STAGE PLANS:
                   Statistics: Num rows: 205 Data size: 19475 Basic stats: COMPLETE Column
stats: COMPLETE
                   value expressions: _col0 (type: string)
         Reducer 3 
-            Execution mode: llap
+            Execution mode: uber
             Reduce Operator Tree:
               Select Operator
                 expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
@@ -462,7 +462,7 @@ STAGE PLANS:
                   Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column
stats: COMPLETE
                   value expressions: _col0 (type: string), _col1 (type: string), _col2 (type:
string)
         Reducer 3 
-            Execution mode: llap
+            Execution mode: uber
             Reduce Operator Tree:
               Select Operator
                 expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2
(type: string), KEY.reducesinkkey0 (type: string)
@@ -551,7 +551,7 @@ STAGE PLANS:
                   Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column
stats: COMPLETE
                   value expressions: _col0 (type: string), _col1 (type: string), _col2 (type:
string)
         Reducer 3 
-            Execution mode: llap
+            Execution mode: uber
             Reduce Operator Tree:
               Select Operator
                 expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2
(type: string), KEY.reducesinkkey0 (type: string)
@@ -991,7 +991,7 @@ STAGE PLANS:
                   Statistics: Num rows: 1219 Data size: 433964 Basic stats: COMPLETE Column
stats: COMPLETE
                   value expressions: _col0 (type: string), _col1 (type: string), _col2 (type:
string)
         Reducer 3 
-            Execution mode: llap
+            Execution mode: uber
             Reduce Operator Tree:
               Select Operator
                 expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2
(type: string), KEY.reducesinkkey0 (type: string)
@@ -1055,7 +1055,7 @@ STAGE PLANS:
                           value expressions: _col0 (type: bigint)
             Execution mode: llap
         Reducer 2 
-            Execution mode: llap
+            Execution mode: uber
             Reduce Operator Tree:
               Group By Operator
                 aggregations: sum(VALUE._col0)



Mime
View raw message