hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1619565 - in /hive/branches/cbo/ql/src: java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/ test/results/clientpositive/
Date Thu, 21 Aug 2014 20:26:45 GMT
Author: hashutosh
Date: Thu Aug 21 20:26:45 2014
New Revision: 1619565

URL: http://svn.apache.org/r1619565
Log:
HIVE-7827 : [CBO] null expr in select list is not handled correctly (Ashutosh Chauhan)

Modified:
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
    hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTBuilder.java
Thu Aug 21 20:26:45 2014
@@ -108,7 +108,7 @@ class ASTBuilder {
 
   static ASTNode selectExpr(ASTNode expr, String alias) {
     return ASTBuilder.construct(HiveParser.TOK_SELEXPR, "TOK_SELEXPR").add(expr)
-        .add(HiveParser.Identifier, alias).node();
+      .add(HiveParser.Identifier, alias).node();
   }
 
   static ASTNode literal(RexLiteral literal) {
@@ -142,6 +142,9 @@ class ASTBuilder {
       type = ((Boolean) val).booleanValue() ? HiveParser.KW_TRUE
           : HiveParser.KW_FALSE;
       break;
+    case NULL:
+      type = HiveParser.TOK_NULL;
+      break;
 
     default:
       throw new RuntimeException("Unsupported Type: " + sqlType);

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
Thu Aug 21 20:26:45 2014
@@ -123,15 +123,23 @@ public class ASTConverter {
     /*
      * 6. Project
      */
-    int i = 0;
-    ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT");
+    if (!select.getChildExps().isEmpty()) {
 
-    for (RexNode r : select.getChildExps()) {
-      ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(new RexVisitor(schema)), select
-          .getRowType().getFieldNames().get(i++));
-      b.add(selectExpr);
+      ASTBuilder b = ASTBuilder.construct(HiveParser.TOK_SELECT, "TOK_SELECT");
+      int i = 0;
+
+      for (RexNode r : select.getChildExps()) {
+        ASTNode selectExpr = ASTBuilder.selectExpr(r.accept(new RexVisitor(schema)), select
+            .getRowType().getFieldNames().get(i++));
+        b.add(selectExpr);
+      }
+      hiveAST.select = b.node();
+    } else {
+      //TODO: We should never be here. But we will be for select null from t1.
+      // Once you figure out why, uncomment following line:
+      // throw new IllegalStateException("why am I here?");
     }
-    hiveAST.select = b.node();
+
 
     /*
      * 7. Order Use in Order By from the block above. RelNode has no pointer to
@@ -347,13 +355,13 @@ public class ASTConverter {
       ASTNode wRangeAst = null;
 
       ASTNode startAST = null;
-      RexWindowBound ub = (RexWindowBound) window.getUpperBound();
+      RexWindowBound ub = window.getUpperBound();
       if (ub != null) {
         startAST = getWindowBound(ub);
       }
 
       ASTNode endAST = null;
-      RexWindowBound lb = (RexWindowBound) window.getLowerBound();
+      RexWindowBound lb = window.getLowerBound();
       if (lb != null) {
         endAST = getWindowBound(lb);
       }
@@ -496,7 +504,7 @@ public class ASTConverter {
      * 1. ProjectRel will always be child of SortRel.<br>
      * 2. In Optiq every projection in ProjectRelBase is uniquely named
      * (unambigous) without using table qualifier (table name).<br>
-     * 
+     *
      * @param order
      *          Hive Sort Rel Node
      * @return Schema

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/RexNodeConverter.java
Thu Aug 21 20:26:45 2014
@@ -15,11 +15,11 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToBinary;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToChar;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToDate;
@@ -40,7 +40,6 @@ import org.eigenbase.rex.RexNode;
 import org.eigenbase.rex.RexUtil;
 import org.eigenbase.sql.SqlOperator;
 import org.eigenbase.sql.fun.SqlCastFunction;
-import org.eigenbase.sql.type.SqlTypeName;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList.Builder;
@@ -81,6 +80,10 @@ public class RexNodeConverter {
   }
 
   public RexNode convert(ExprNodeDesc expr) throws SemanticException {
+    if (expr instanceof ExprNodeNullDesc) {
+      return m_cluster.getRexBuilder().makeNullLiteral(TypeConverter.convert(
+        expr.getTypeInfo(), m_cluster.getRexBuilder().getTypeFactory()).getSqlTypeName());
+    }
     if (expr instanceof ExprNodeGenericFuncDesc) {
       return convert((ExprNodeGenericFuncDesc) expr);
     } else if (expr instanceof ExprNodeConstantDesc) {
@@ -90,8 +93,7 @@ public class RexNodeConverter {
     } else {
       throw new RuntimeException("Unsupported Expression");
     }
-    // TODO: handle a) ExprNodeNullDesc b) ExprNodeFieldDesc c)
-    // ExprNodeColumnListDesc
+    // TODO: handle a) ExprNodeFieldDesc b) ExprNodeColumnListDesc
   }
 
   private RexNode convert(final ExprNodeGenericFuncDesc func) throws SemanticException {
@@ -230,9 +232,10 @@ public class RexNodeConverter {
     RelDataType optiqDataType = TypeConverter.convert(hiveType, dtFactory);
 
     PrimitiveCategory hiveTypeCategory = hiveType.getPrimitiveCategory();
-    RexNode optiqLiteral = null;
+
     Object value = literal.getValue();
 
+    RexNode optiqLiteral = null;
     // TODO: Verify if we need to use ConstantObjectInspector to unwrap data
     switch (hiveTypeCategory) {
     case BOOLEAN:

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
(original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/TypeConverter.java
Thu Aug 21 20:26:45 2014
@@ -110,8 +110,7 @@ public class TypeConverter {
 
     switch (type.getPrimitiveCategory()) {
     case VOID:
-      // @todo: followup on VOID type in hive
-      convertedType = dtFactory.createSqlType(SqlTypeName.OTHER);
+      convertedType = dtFactory.createSqlType(SqlTypeName.NULL);
       break;
     case BOOLEAN:
       convertedType = dtFactory.createSqlType(SqlTypeName.BOOLEAN);
@@ -135,6 +134,7 @@ public class TypeConverter {
       convertedType = dtFactory.createSqlType(SqlTypeName.DOUBLE);
       break;
     case STRING:
+      //TODO: shall we pass -1 for len to distinguish between STRING & VARCHAR on way
out
       convertedType = dtFactory.createSqlType(SqlTypeName.VARCHAR, 1);
       break;
     case DATE:
@@ -163,6 +163,10 @@ public class TypeConverter {
       break;
     }
 
+    if (null == convertedType) {
+      throw new RuntimeException("Unsupported Type : " + type.getTypeName());
+    }
+
     return convertedType;
   }
 
@@ -184,6 +188,7 @@ public class TypeConverter {
     List<RelDataType> fTypes = Lists.transform(
         structType.getAllStructFieldTypeInfos(),
         new Function<TypeInfo, RelDataType>() {
+          @Override
           public RelDataType apply(TypeInfo tI) {
             return convert(tI, dtFactory);
           }
@@ -197,7 +202,7 @@ public class TypeConverter {
     // @todo what do we about unions?
     throw new UnsupportedOperationException();
   }
-  
+
   public static TypeInfo convert(RelDataType rType) {
     if ( rType.isStruct() ) {
       return convertStructType(rType);
@@ -209,11 +214,12 @@ public class TypeConverter {
       return convertPrimtiveType(rType);
     }
   }
-  
+
   public static TypeInfo convertStructType(RelDataType rType) {
     List<TypeInfo> fTypes = Lists.transform(
         rType.getFieldList(),
         new Function<RelDataTypeField, TypeInfo>() {
+          @Override
           public TypeInfo apply(RelDataTypeField f) {
             return convert(f.getType());
           }
@@ -221,22 +227,23 @@ public class TypeConverter {
     List<String> fNames = Lists.transform(
         rType.getFieldList(),
         new Function<RelDataTypeField, String>() {
+          @Override
           public String apply(RelDataTypeField f) {
             return f.getName();
           }
         });
     return TypeInfoFactory.getStructTypeInfo(fNames, fTypes);
   }
-  
+
   public static TypeInfo convertMapType(RelDataType rType) {
-    return TypeInfoFactory.getMapTypeInfo(convert(rType.getKeyType()), 
+    return TypeInfoFactory.getMapTypeInfo(convert(rType.getKeyType()),
         convert(rType.getValueType()));
   }
-  
+
   public static TypeInfo convertListType(RelDataType rType) {
     return TypeInfoFactory.getListTypeInfo(convert(rType.getComponentType()));
   }
-  
+
   public static TypeInfo convertPrimtiveType(RelDataType rType) {
     switch(rType.getSqlTypeName()) {
     case BOOLEAN:
@@ -269,7 +276,7 @@ public class TypeConverter {
       default:
       return TypeInfoFactory.voidTypeInfo;
     }
-    
+
   }
 
   /*********************** Convert Optiq Types To Hive Types ***********************/

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu
Aug 21 20:26:45 2014
@@ -13147,14 +13147,8 @@ public class SemanticAnalyzer extends Ba
       }
       selectStar = selectStar && exprList.getChildCount() == posn + 1;
 
-      // 7. Replace NULL with CAST(NULL AS STRING)
       ArrayList<String> columnNames = new ArrayList<String>();
       for (int i = 0; i < col_list.size(); i++) {
-        // Replace NULL with CAST(NULL AS STRING)
-        if (col_list.get(i) instanceof ExprNodeNullDesc) {
-          col_list.set(i, new ExprNodeConstantDesc(
-              TypeInfoFactory.stringTypeInfo, null));
-        }
         columnNames.add(getColumnInternalName(i));
       }
 

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q Thu Aug 21 20:26:45
2014
@@ -215,4 +215,5 @@ select * from (select c_int, b, t1.c fro
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1
>= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having
t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order
by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from
t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float,
t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int +
c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where
(b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1
or c >= 1) and (a + c) >= 0 order by c, a;
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1
>= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having
t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order
by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as
r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group
by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and
(c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3
on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a >
0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
 
-
+-- 13. null expr in select list
+select null from t3;

Modified: hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out?rev=1619565&r1=1619564&r2=1619565&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out Thu Aug 21
20:26:45 2014
@@ -17849,3 +17849,33 @@ POSTHOOK: Input: default@t3
  1 	2
 1 	2
 1	12
+PREHOOK: query: -- 13. null expr in select list
+select null from t3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: -- 13. null expr in select list
+select null from t3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL
+NULL



Mime
View raw message