hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [20/27] hive git commit: HIVE-12857 : LLAP: modify the decider to allow using LLAP with whitelisted UDFs (Sergey Shelukhin, reviewed by Gunther Hagleitner)
Date Fri, 26 Feb 2016 23:16:03 GMT
HIVE-12857 : LLAP: modify the decider to allow using LLAP with whitelisted UDFs (Sergey Shelukhin,
reviewed by Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e44198f5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e44198f5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e44198f5

Branch: refs/heads/llap
Commit: e44198f5e6b2b5a83c34d70bd12bb4e772f737dd
Parents: be00006
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Thu Feb 25 19:41:56 2016 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Thu Feb 25 19:41:56 2016 -0800

----------------------------------------------------------------------
 itests/qtest/pom.xml                            |   4 +-
 .../test/resources/testconfiguration.properties |   6 +-
 .../hadoop/hive/ql/exec/FunctionRegistry.java   |  15 +
 .../hadoop/hive/ql/exec/FunctionTask.java       |  10 +-
 .../apache/hadoop/hive/ql/exec/Registry.java    |  44 +++
 .../hive/ql/optimizer/physical/LlapDecider.java |  50 +--
 ql/src/test/queries/clientpositive/llap_udf.q   |  47 +++
 .../results/clientpositive/llap/llap_udf.q.out  | 307 +++++++++++++++++++
 8 files changed, 458 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 292e562..cb68e39 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -431,7 +431,7 @@
                   templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
                   queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
                   queryFile="${qfile}"
-                  excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files},${disabled.query.files}"
+                  excludeQueryFile="${minillap.query.files},${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files},${disabled.query.files}"
                   queryFileRegex="${qfile_regex}"
                   clusterMode="${clustermode}"
                   runDisabled="${run_disabled}"
@@ -535,7 +535,7 @@
                   templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
                   queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
                   queryFile="${qfile}"
-                  includeQueryFile="${minillap.query.files}"
+                  includeQueryFile="${minillap.query.files},${minillap.shared.query.files}"
                   queryFileRegex="${qfile_regex}"
                   clusterMode="llap"
                   runDisabled="${run_disabled}"

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 58d0a45..68dbd0c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -445,7 +445,7 @@ minitez.query.files=bucket_map_join_tez1.q,\
 
 
 
-minillap.query.files=bucket_map_join_tez1.q,\
+minillap.shared.query.files=bucket_map_join_tez1.q,\
   bucket_map_join_tez2.q,\
   constprog_dpp.q,\
   cte_1.q,\
@@ -496,6 +496,10 @@ minillap.query.files=bucket_map_join_tez1.q,\
   tez_join.q,\
   tez_union_multiinsert.q
 
+
+minillap.query.files=llap_udf.q
+
+
 encrypted.query.files=encryption_join_unencrypted_tbl.q,\
   encryption_insert_partition_static.q,\
   encryption_insert_partition_dynamic.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index e0e030f..d1e1441 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -1480,6 +1480,21 @@ public final class FunctionRegistry {
     return system.registerPermanentFunction(functionName, className, registerToSession, resources);
   }
 
+  public static boolean isPermanentFunction(ExprNodeGenericFuncDesc fnExpr) {
+    GenericUDF udf = fnExpr.getGenericUDF();
+    if (udf == null) return false;
+
+    Class<?> clazz = udf.getClass();
+    if (udf instanceof GenericUDFBridge) {
+      clazz = ((GenericUDFBridge)udf).getUdfClass();
+    }
+
+    if (clazz != null) {
+      return system.isPermanentFunc(clazz);
+    }
+    return false;
+  }
+
   public static void unregisterPermanentFunction(String functionName) throws HiveException
{
     system.unregisterFunction(functionName);
     unregisterTemporaryUDF(functionName);

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
index 77f11b9..1b971fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
@@ -136,8 +136,16 @@ public class FunctionTask extends Task<FunctionWork> {
     // For permanent functions, check for any resources from local filesystem.
     checkLocalFunctionResources(db, createFunctionDesc.getResources());
 
-    FunctionInfo registered = FunctionRegistry.registerPermanentFunction(
+    FunctionInfo registered = null;
+    try {
+      registered = FunctionRegistry.registerPermanentFunction(
         registeredName, className, true, toFunctionResource(resources));
+    } catch (RuntimeException ex) {
+      Throwable t = ex;
+      while (t.getCause() != null) {
+        t = t.getCause();
+      }
+    }
     if (registered == null) {
       console.printError("Failed to register " + registeredName
           + " using class " + createFunctionDesc.getClassName());

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index 16c8c8f..d5f4a37 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -55,6 +55,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
@@ -71,6 +72,11 @@ public class Registry {
    */
   private final Map<String, FunctionInfo> mFunctions = new LinkedHashMap<String,
FunctionInfo>();
   private final Set<Class<?>> builtIns = Collections.synchronizedSet(new HashSet<Class<?>>());
+  /**
+   * Persistent map contains refcounts that are only modified in synchronized methods for
now,
+   * so there's no separate effort to make refcount operations thread-safe.
+   */
+  private final Map<Class<?>, Integer> persistent = new ConcurrentHashMap<>();
   private final Set<ClassLoader> mSessionUDFLoaders = new LinkedHashSet<ClassLoader>();
 
   private final boolean isNative;
@@ -285,6 +291,10 @@ public class Registry {
     return udfClass != null && builtIns.contains(udfClass);
   }
 
+  public boolean isPermanentFunc(Class<?> udfClass) {
+    return udfClass != null && persistent.containsKey(udfClass);
+  }
+
   public synchronized Set<String> getCurrentFunctionNames() {
     return getFunctionNames((Pattern)null);
   }
@@ -415,9 +425,27 @@ public class Registry {
     mFunctions.put(functionName, function);
     if (function.isBuiltIn()) {
       builtIns.add(function.getFunctionClass());
+    } else if (function.isPersistent()) {
+      Class<?> functionClass = getPermanentUdfClass(function);
+      Integer refCount = persistent.get(functionClass);
+      persistent.put(functionClass, Integer.valueOf(refCount == null ? 1 : refCount + 1));
     }
   }
 
+  private Class<?> getPermanentUdfClass(FunctionInfo function) {
+    Class<?> functionClass = function.getFunctionClass();
+    if (functionClass == null) {
+      // Expected for permanent UDFs at this point.
+      ClassLoader loader = Utilities.getSessionSpecifiedClassLoader();
+      try {
+        functionClass = Class.forName(function.getClassName(), true, loader);
+      } catch (ClassNotFoundException ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+    return functionClass;
+  }
+
   public synchronized void unregisterFunction(String functionName) throws HiveException {
     functionName = functionName.toLowerCase();
     FunctionInfo fi = mFunctions.get(functionName);
@@ -427,6 +455,21 @@ public class Registry {
       }
       mFunctions.remove(functionName);
       fi.discarded();
+      if (fi.isPersistent()) {
+        removePersistentFunction(fi);
+      }
+    }
+  }
+
+  /** Should only be called from synchronized methods. */
+  private void removePersistentFunction(FunctionInfo fi) {
+    Class<?> functionClass = getPermanentUdfClass(fi);
+    Integer refCount = persistent.get(functionClass);
+    assert refCount != null;
+    if (refCount == 1) {
+      persistent.remove(functionClass);
+    } else {
+      persistent.put(functionClass, Integer.valueOf(refCount - 1));
     }
   }
 
@@ -517,6 +560,7 @@ public class Registry {
     }
     mFunctions.clear();
     builtIns.clear();
+    persistent.clear();
   }
 
   public synchronized void closeCUDFLoaders() {

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index f2998af..80dbd31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -101,15 +101,18 @@ public class LlapDecider implements PhysicalPlanResolver {
   class LlapDecisionDispatcher implements Dispatcher {
     private final HiveConf conf;
     private final boolean doSkipUdfCheck;
+    private final boolean arePermanentFnsAllowed;
 
     public LlapDecisionDispatcher(PhysicalContext pctx) {
       conf = pctx.getConf();
       doSkipUdfCheck = HiveConf.getBoolVar(conf, ConfVars.LLAP_SKIP_COMPILE_UDF_CHECK);
+      arePermanentFnsAllowed = HiveConf.getBoolVar(conf, ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS);
     }
 
     @Override
     public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs)
       throws SemanticException {
+      @SuppressWarnings("unchecked")
       Task<? extends Serializable> currTask = (Task<? extends Serializable>)
nd;
       if (currTask instanceof TezTask) {
         TezWork work = ((TezTask) currTask).getWork();
@@ -237,10 +240,20 @@ public class LlapDecider implements PhysicalPlanResolver {
           exprs.addAll(cur.getChildren());
         }
 
-        if (!doSkipUdfCheck && cur instanceof ExprNodeGenericFuncDesc
-            && !FunctionRegistry.isBuiltInFuncExpr((ExprNodeGenericFuncDesc)cur))
{
-          LOG.info("Not a built-in function: " + cur.getExprString());
-          return false;
+        if (!doSkipUdfCheck && cur instanceof ExprNodeGenericFuncDesc) {
+          ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc)cur;
+          boolean isBuiltIn = FunctionRegistry.isBuiltInFuncExpr(funcDesc);
+          if (!isBuiltIn) {
+            if (!arePermanentFnsAllowed) {
+              LOG.info("Not a built-in function: " + cur.getExprString()
+                + " (permanent functions are disabled)");
+              return false;
+            }
+            if (!FunctionRegistry.isPermanentFunction(funcDesc)) {
+              LOG.info("Not a built-in or permanent function: " + cur.getExprString());
+              return false;
+            }
+          }
         }
       }
       return true;
@@ -261,24 +274,22 @@ public class LlapDecider implements PhysicalPlanResolver {
     }
 
     private boolean checkExpressions(Collection<ExprNodeDesc> exprs) {
-      boolean result = true;
-      for (ExprNodeDesc expr: exprs) {
-        result = result && checkExpression(expr);
+      for (ExprNodeDesc expr : exprs) {
+        if (!checkExpression(expr)) return false;
       }
-      return result;
+      return true;
     }
 
     private boolean checkAggregators(Collection<AggregationDesc> aggs) {
-      boolean result = true;
       try {
-	for (AggregationDesc agg: aggs) {
-	  result = result && checkAggregator(agg);
-	}
+        for (AggregationDesc agg: aggs) {
+          if (!checkAggregator(agg)) return false;
+        }
       } catch (SemanticException e) {
-	LOG.warn("Exception testing aggregators.",e);
-	result = false;
+        LOG.warn("Exception testing aggregators.",e);
+        return false;
       }
-      return result;
+      return true;
     }
 
     private Map<Rule, NodeProcessor> getRules() {
@@ -291,8 +302,7 @@ public class LlapDecider implements PhysicalPlanResolver {
             return new Boolean(false);
           }
         });
-      opRules.put(new RuleRegExp("No user code in fil",
-              FilterOperator.getOperatorName() + "%"),
+      opRules.put(new RuleRegExp("No user code in fil", FilterOperator.getOperatorName()
+ "%"),
           new NodeProcessor() {
           @Override
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
@@ -301,8 +311,7 @@ public class LlapDecider implements PhysicalPlanResolver {
             return new Boolean(checkExpression(expr));
           }
         });
-      opRules.put(new RuleRegExp("No user code in gby",
-              GroupByOperator.getOperatorName() + "%"),
+      opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName()
+ "%"),
           new NodeProcessor() {
           @Override
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
@@ -311,8 +320,7 @@ public class LlapDecider implements PhysicalPlanResolver {
             return new Boolean(checkAggregators(aggs));
           }
         });
-      opRules.put(new RuleRegExp("No user code in select",
-              SelectOperator.getOperatorName() + "%"),
+      opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName()
+ "%"),
           new NodeProcessor() {
           @Override
           public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/test/queries/clientpositive/llap_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q
new file mode 100644
index 0000000..c964f2b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/llap_udf.q
@@ -0,0 +1,47 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.execution.mode=llap;
+set hive.llap.execution.mode=all;
+set hive.fetch.task.conversion=none;
+set hive.llap.daemon.allow.permanent.fns=true;
+
+drop table if exists src_orc;
+create table src_orc stored as orc as select * from src;
+
+-- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin
+
+CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
+
+EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc;
+
+CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
+CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
+CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
+
+EXPLAIN
+SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key
as string)) from src_orc;
+
+-- Verification is based on classes, so 0 would work based on 4.
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc;
+
+DROP FUNCTION test_udf2;
+
+-- ...verify that 3 still works
+EXPLAIN
+SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;
+
+DROP FUNCTION test_udf4;
+
+-- ...now 0 should stop working
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc;
+
+set hive.llap.daemon.allow.permanent.fns=false;
+
+EXPLAIN
+SELECT test_udf3(cast(key as string)) from src_orc;
+
+
+drop table if exists src_orc;
+set hive.execution.mode=container;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/test/results/clientpositive/llap/llap_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
new file mode 100644
index 0000000..29ed978
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
@@ -0,0 +1,307 @@
+PREHOOK: query: drop table if exists src_orc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists src_orc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table src_orc stored as orc as select * from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: create table src_orc stored as orc as select * from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_orc
+PREHOOK: query: -- Not using GenericUDFTestGetJavaBoolean; that is already registered when
tests begin
+
+CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_udf0
+POSTHOOK: query: -- Not using GenericUDFTestGetJavaBoolean; that is already registered when
tests begin
+
+CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_udf0
+PREHOOK: query: EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_orc
+                  Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: evaluate_npe(key) (type: string)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf2
+POSTHOOK: query: CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf2
+PREHOOK: query: CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf3
+POSTHOOK: query: CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf3
+PREHOOK: query: CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf4
+POSTHOOK: query: CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf4
+PREHOOK: query: EXPLAIN
+SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key
as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key
as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_orc
+                  Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: GenericUDFTestGetJavaString(key) (type: string), GenericUDFTestGetJavaString(key)
(type: string), evaluate_npe(key) (type: string)
+                    outputColumnNames: _col0, _col1, _col2
+                    Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+            Execution mode: llap
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: -- Verification is based on classes, so 0 would work based on 4.
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Verification is based on classes, so 0 would work based on 4.
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_orc
+                  Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: evaluate_npe(key) (type: string)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+            Execution mode: llap
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: DROP FUNCTION test_udf2
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf2
+POSTHOOK: query: DROP FUNCTION test_udf2
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf2
+PREHOOK: query: -- ...verify that 3 still works
+EXPLAIN
+SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: -- ...verify that 3 still works
+EXPLAIN
+SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_orc
+                  Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: GenericUDFTestGetJavaString(key) (type: string), evaluate_npe(key)
(type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+            Execution mode: llap
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: DROP FUNCTION test_udf4
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf4
+POSTHOOK: query: DROP FUNCTION test_udf4
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf4
+PREHOOK: query: -- ...now 0 should stop working
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: -- ...now 0 should stop working
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_orc
+                  Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: evaluate_npe(key) (type: string)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT test_udf3(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT test_udf3(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_orc
+                  Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    expressions: GenericUDFTestGetJavaString(key) (type: string)
+                    outputColumnNames: _col0
+                    Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column
stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: drop table if exists src_orc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_orc
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: drop table if exists src_orc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_orc
+POSTHOOK: Output: default@src_orc


Mime
View raw message