hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1625868 - in /hive/branches/cbo/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ test/queries/clientpositive/ test/results/clientpositive/
Date Thu, 18 Sep 2014 00:56:47 GMT
Author: sershe
Date: Thu Sep 18 00:56:46 2014
New Revision: 1625868

URL: http://svn.apache.org/r1625868
Log:
HIVE-8080: CBO: function name may not match UDF name during translation (Sergey Shelukhin,
reviewed by Laljo John Pullokkaran)

Modified:
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
    hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q
    hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Thu
Sep 18 00:56:46 2014
@@ -639,6 +639,14 @@ public final class FunctionRegistry {
     }
   }
 
+  public static String getNormalizedFunctionName(String fn) {
+    // Does the same thing as getFunctionInfo, except for getting the function info.
+    fn = fn.toLowerCase();
+    return (FunctionUtils.isQualifiedFunctionName(fn) || mFunctions.get(fn) != null) ? fn
+        : FunctionUtils.qualifyFunctionName(
+            fn, SessionState.get().getCurrentDatabase().toLowerCase());
+  }
+
   private static <T extends CommonFunctionInfo> T getFunctionInfo(
       Map<String, T> mFunctions, String functionName) {
     functionName = functionName.toLowerCase();

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
Thu Sep 18 00:56:46 2014
@@ -152,9 +152,7 @@ public class RexNodeConverter {
     List<RexNode> childRexNodeLst = new LinkedList<RexNode>();
     Builder<RelDataType> argTypeBldr = ImmutableList.<RelDataType> builder();
 
-    // TODO: 1) Expand to other functions as needed 2) What about types
-    // other
-    // than primitive
+    // TODO: 1) Expand to other functions as needed 2) What about types other than primitive.
     if (func.getGenericUDF() instanceof GenericUDFBaseNumeric) {
       tgtDT = func.getTypeInfo();
     } else if (func.getGenericUDF() instanceof GenericUDFBaseCompare) {
@@ -183,8 +181,8 @@ public class RexNodeConverter {
     if (expr == null) {
       retType = (expr != null) ? expr.getType() : TypeConverter.convert(func.getTypeInfo(),
           m_cluster.getTypeFactory());
-      SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(func.getGenericUDF(),
-          argTypeBldr.build(), retType);
+      SqlOperator optiqOp = SqlFunctionConverter.getOptiqOperator(
+          func.getFuncText(), func.getGenericUDF(), argTypeBldr.build(), retType);
       expr = m_cluster.getRexBuilder().makeCall(optiqOp, childRexNodeLst);
     } else {
       retType = expr.getType();

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/SqlFunctionConverter.java
Thu Sep 18 00:56:46 2014
@@ -21,6 +21,9 @@ import java.lang.annotation.Annotation;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.FunctionInfo;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -61,26 +64,36 @@ import com.google.common.collect.Immutab
 import com.google.common.collect.Maps;
 
 public class SqlFunctionConverter {
+  private static final Log LOG = LogFactory.getLog(SqlFunctionConverter.class);
+
   static final Map<String, SqlOperator>    hiveToOptiq;
   static final Map<SqlOperator, HiveToken> optiqToHiveToken;
   static final Map<SqlOperator, String>    reverseOperatorMap;
 
   static {
-    Builder builder = new Builder();
+    StaticBlockBuilder builder = new StaticBlockBuilder();
     hiveToOptiq = ImmutableMap.copyOf(builder.hiveToOptiq);
     optiqToHiveToken = ImmutableMap.copyOf(builder.optiqToHiveToken);
     reverseOperatorMap = ImmutableMap.copyOf(builder.reverseOperatorMap);
   }
 
-  public static SqlOperator getOptiqOperator(GenericUDF hiveUDF,
+  public static SqlOperator getOptiqOperator(String funcTextName, GenericUDF hiveUDF,
       ImmutableList<RelDataType> optiqArgTypes, RelDataType retType) throws OptiqSemanticException
{
     // handle overloaded methods first
     if (hiveUDF instanceof GenericUDFOPNegative) {
       return SqlStdOperatorTable.UNARY_MINUS;
     } else if (hiveUDF instanceof GenericUDFOPPositive) {
       return SqlStdOperatorTable.UNARY_PLUS;
-    } // do genric lookup
-    return getOptiqFn(getName(hiveUDF), optiqArgTypes, retType);
+    } // do generic lookup
+    String name = null;
+    if (StringUtils.isEmpty(funcTextName)) {
+      name = getName(hiveUDF); // this should probably never happen, see getName comment
+      LOG.warn("The function text was empty, name from annotation is " + name);
+    } else {
+      // We could just do toLowerCase here and let SA qualify it, but let's be proper...
+      name = FunctionRegistry.getNormalizedFunctionName(funcTextName);
+    }
+    return getOptiqFn(name, optiqArgTypes, retType);
   }
 
   public static GenericUDF getHiveUDF(SqlOperator op, RelDataType dt) {
@@ -197,6 +210,9 @@ public class SqlFunctionConverter {
 
   }
 
+  // TODO: this is not valid. Function names for built-in UDFs are specified in FunctionRegistry,
+  //       and only happen to match annotations. For user UDFs, the name is what user specifies
at
+  //       creation time (annotation can be absent, different, or duplicate some other function).
   private static String getName(GenericUDF hiveUDF) {
     String udfName = null;
     if (hiveUDF instanceof GenericUDFBridge) {
@@ -228,12 +244,13 @@ public class SqlFunctionConverter {
     return udfName;
   }
 
-  private static class Builder {
+  /** This class is used to build immutable hashmaps in the static block above. */
+  private static class StaticBlockBuilder {
     final Map<String, SqlOperator>    hiveToOptiq        = Maps.newHashMap();
     final Map<SqlOperator, HiveToken> optiqToHiveToken   = Maps.newHashMap();
     final Map<SqlOperator, String>    reverseOperatorMap = Maps.newHashMap();
 
-    Builder() {
+    StaticBlockBuilder() {
       registerFunction("+", SqlStdOperatorTable.PLUS, hToken(HiveParser.PLUS, "+"));
       registerFunction("-", SqlStdOperatorTable.MINUS, hToken(HiveParser.MINUS, "-"));
       registerFunction("*", SqlStdOperatorTable.MULTIPLY, hToken(HiveParser.STAR, "*"));

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/create_func1.q Thu Sep 18 00:56:46
2014
@@ -2,11 +2,16 @@
 -- qtest_get_java_boolean should already be created during test initialization
 select qtest_get_java_boolean('true'), qtest_get_java_boolean('false') from src limit 1;
 
+describe function extended qtest_get_java_boolean;
+
 create database mydb;
 create function mydb.func1 as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper';
 
 show functions mydb.func1;
 
+describe function extended mydb.func1;
+
+
 select mydb.func1('abc') from src limit 1;
 
 drop function mydb.func1;

Modified: hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out?rev=1625868&r1=1625867&r2=1625868&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/create_func1.q.out Thu Sep 18 00:56:46
2014
@@ -9,6 +9,12 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 #### A masked pattern was here ####
 true	false
+PREHOOK: query: describe function extended qtest_get_java_boolean
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function extended qtest_get_java_boolean
+POSTHOOK: type: DESCFUNCTION
+qtest_get_java_boolean(str) - GenericUDF to return native Java's boolean type
+Synonyms: default.qtest_get_java_boolean
 PREHOOK: query: create database mydb
 PREHOOK: type: CREATEDATABASE
 PREHOOK: Output: database:mydb
@@ -28,6 +34,15 @@ PREHOOK: type: SHOWFUNCTIONS
 POSTHOOK: query: show functions mydb.func1
 POSTHOOK: type: SHOWFUNCTIONS
 mydb.func1
+PREHOOK: query: describe function extended mydb.func1
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: describe function extended mydb.func1
+POSTHOOK: type: DESCFUNCTION
+mydb.func1(str) - Returns str with all characters changed to uppercase
+Synonyms: upper, ucase
+Example:
+  > SELECT mydb.func1('Facebook') FROM src LIMIT 1;
+  'FACEBOOK'
 PREHOOK: query: select mydb.func1('abc') from src limit 1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src



Mime
View raw message