hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vik...@apache.org
Subject hive git commit: HIVE-13342: Improve logging in llap decider and throw exception in case llap mode is all but we cannot run in llap. (Vikram Dixit K, reviewed by Sergey Shelukhin and Gunther Hagleitner)
Date Tue, 10 May 2016 21:04:27 GMT
Repository: hive
Updated Branches:
  refs/heads/master e3b6c9b1e -> e087ba845


HIVE-13342: Improve logging in llap decider and throw exception in case llap mode is all but
we cannot run in llap. (Vikram Dixit K, reviewed by Sergey Shelukhin and Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e087ba84
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e087ba84
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e087ba84

Branch: refs/heads/master
Commit: e087ba84537555eb37b0047e7ace3f1b9dc06d08
Parents: e3b6c9b
Author: vikram <vikram@hortonworks.com>
Authored: Tue May 10 14:01:30 2016 -0700
Committer: vikram <vikram@hortonworks.com>
Committed: Tue May 10 14:03:29 2016 -0700

----------------------------------------------------------------------
 .../hive/ql/optimizer/physical/LlapDecider.java | 37 +++++++++++++++-----
 ql/src/test/queries/clientpositive/llap_udf.q   |  5 +++
 .../results/clientpositive/llap/llap_udf.q.out  |  2 --
 3 files changed, 34 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index c0b7a32..e6f8f75 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -132,6 +132,12 @@ public class LlapDecider implements PhysicalPlanResolver {
       throws SemanticException {
       if (evaluateWork(tezWork, work)) {
         convertWork(tezWork, work);
+      } else {
+        if (mode == all) {
+          throw new SemanticException("Llap mode is set to all but cannot run work in llap
mode." +
+              "Set " + HiveConf.ConfVars.LLAP_EXECUTION_MODE + " = auto or set " +
+              HiveConf.ConfVars.HIVE_EXECUTION_MODE + " = container");
+        }
       }
     }
 
@@ -228,6 +234,7 @@ public class LlapDecider implements PhysicalPlanResolver {
       }
 
       // couldn't convince you otherwise? well then let's llap.
+      LOG.info("Can run work " + work.getName() + " in llap mode.");
       return true;
     }
 
@@ -304,6 +311,7 @@ public class LlapDecider implements PhysicalPlanResolver {
           @Override
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
               Object... os) {
+            LOG.debug("Cannot run operator [" + n + "] in llap mode.");
             return new Boolean(false);
           }
         });
@@ -313,7 +321,11 @@ public class LlapDecider implements PhysicalPlanResolver {
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
               Object... os) {
             ExprNodeDesc expr = ((FilterOperator)n).getConf().getPredicate();
-            return new Boolean(checkExpression(expr));
+            Boolean retval = new Boolean(checkExpression(expr));
+            if (!retval) {
+              LOG.info("Cannot run filter operator [" + n + "] in llap mode");
+            }
+            return new Boolean(retval);
           }
         });
       opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName()
+ "%"),
@@ -321,9 +333,13 @@ public class LlapDecider implements PhysicalPlanResolver {
           @Override
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
               Object... os) {
-              @SuppressWarnings("unchecked")
-              List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators();
-            return new Boolean(checkAggregators(aggs));
+            @SuppressWarnings("unchecked")
+            List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators();
+            Boolean retval = new Boolean(checkAggregators(aggs));
+            if (!retval) {
+              LOG.info("Cannot run group by operator [" + n + "] in llap mode");
+            }
+            return new Boolean(retval);
           }
         });
       opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName()
+ "%"),
@@ -331,9 +347,13 @@ public class LlapDecider implements PhysicalPlanResolver {
           @Override
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
               Object... os) {
-              @SuppressWarnings({ "unchecked" })
-              List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList();
-            return new Boolean(checkExpressions(exprs));
+            @SuppressWarnings({ "unchecked" })
+            List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList();
+            Boolean retval = new Boolean(checkExpressions(exprs));
+            if (!retval) {
+              LOG.info("Cannot run select operator [" + n + "] in llap mode");
+            }
+            return new Boolean(retval);
           }
         });
 
@@ -355,6 +375,7 @@ public class LlapDecider implements PhysicalPlanResolver {
       for (Node n : nodeOutput.keySet()) {
         if (nodeOutput.get(n) != null) {
           if (!((Boolean)nodeOutput.get(n))) {
+            LOG.info("Cannot run in LLAP mode.");
             return false;
           }
         }
@@ -379,7 +400,7 @@ public class LlapDecider implements PhysicalPlanResolver {
           Arrays.asList(pd.getInputFileFormatClass().getInterfaces());
         if (!interfaceList.contains(VectorizedInputFormatInterface.class)) {
           LOG.info("Input format: " + pd.getInputFileFormatClassName()
-              + ", doesn't provide vectorized input");
+            + ", doesn't provide vectorized input");
           return false;
         }
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/test/queries/clientpositive/llap_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q
index 2224bd5..8b1dcb7 100644
--- a/ql/src/test/queries/clientpositive/llap_udf.q
+++ b/ql/src/test/queries/clientpositive/llap_udf.q
@@ -12,8 +12,10 @@ create table src_orc stored as orc as select * from src;
 
 CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
 
+set hive.llap.execution.mode=auto;
 EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc;
 
+set hive.llap.execution.mode=all;
 CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
 CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
 CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
@@ -21,18 +23,21 @@ CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEv
 EXPLAIN
 SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key
as string)) from src_orc;
 
+set hive.llap.execution.mode=auto;
 -- Verification is based on classes, so 0 would work based on 4.
 EXPLAIN
 SELECT test_udf0(cast(key as string)) from src_orc;
 
 DROP FUNCTION test_udf2;
 
+set hive.llap.execution.mode=all;
 -- ...verify that 3 still works
 EXPLAIN
 SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;
 
 DROP FUNCTION test_udf4;
 
+set hive.llap.execution.mode=auto;
 -- ...now 0 should stop working
 EXPLAIN
 SELECT test_udf0(cast(key as string)) from src_orc;

http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/test/results/clientpositive/llap/llap_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
index 6197cf0..e517942 100644
--- a/ql/src/test/results/clientpositive/llap/llap_udf.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
@@ -157,8 +157,6 @@ STAGE PLANS:
                           input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                           output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                           serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Execution mode: llap
-            LLAP IO: no inputs
 
   Stage: Stage-0
     Fetch Operator


Mime
View raw message