hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1633723 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/plan/ test/queries/clientpositive/ test/results/clientpositive/
Date Wed, 22 Oct 2014 22:21:44 GMT
Author: hashutosh
Date: Wed Oct 22 22:21:44 2014
New Revision: 1633723

URL: http://svn.apache.org/r1633723
Log:
HIVE-8555 : Too many casts results in loss of original string representation for constant
 (Ashutosh Chauhan via Sergey Shelukhin)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
    hive/trunk/ql/src/test/queries/clientpositive/constprog_type.q
    hive/trunk/ql/src/test/results/clientpositive/constprog_type.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1633723&r1=1633722&r2=1633723&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Wed Oct 22
22:21:44 2014
@@ -1739,13 +1739,13 @@ public final class FunctionRegistry {
     if (!(desc instanceof ExprNodeGenericFuncDesc)) {
       return false;
     }
-    GenericUDF genericUDF = ((ExprNodeGenericFuncDesc)desc).getGenericUDF();
-    Class udfClass;
-    if (genericUDF instanceof GenericUDFBridge) {
-      udfClass = ((GenericUDFBridge)genericUDF).getUdfClass();
-    } else {
-      udfClass = genericUDF.getClass();
-    }
+    return isOpCast(((ExprNodeGenericFuncDesc)desc).getGenericUDF());
+  }
+
+  public static boolean isOpCast(GenericUDF genericUDF) {
+    Class udfClass = (genericUDF instanceof GenericUDFBridge) ?
+        ((GenericUDFBridge)genericUDF).getUdfClass() : genericUDF.getClass();
+
     return udfClass == UDFToBoolean.class || udfClass == UDFToByte.class ||
         udfClass == UDFToDouble.class || udfClass == UDFToFloat.class ||
         udfClass == UDFToInteger.class || udfClass == UDFToLong.class ||

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java?rev=1633723&r1=1633722&r2=1633723&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
Wed Oct 22 22:21:44 2014
@@ -166,6 +166,10 @@ public final class ConstantPropagateProc
     }
     LOG.debug("Casting " + desc + " to type " + ti);
     ExprNodeConstantDesc c = (ExprNodeConstantDesc) desc;
+    if (null != c.getFoldedFromVal() && priti.getTypeName().equals(serdeConstants.STRING_TYPE_NAME))
{
+      // avoid double casting to preserve original string representation of constant.
+      return new ExprNodeConstantDesc(c.getFoldedFromVal());
+    }
     ObjectInspector origOI =
         TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(desc.getTypeInfo());
     ObjectInspector oi =
@@ -551,7 +555,12 @@ public final class ConstantPropagateProc
         LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable.");
         return null;
       }
-      return new ExprNodeConstantDesc(o);
+      String constStr = null;
+      if(arguments.length == 1 && FunctionRegistry.isOpCast(udf)) {
+        // remember original string representation of constant.
+        constStr = arguments[0].get().toString();
+      }
+      return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr);
     } catch (HiveException e) {
       LOG.error("Evaluation function " + udf.getClass()
           + " failed in Constant Propagatation Optimizer.");

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1633723&r1=1633722&r2=1633723&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Wed Oct
22 22:21:44 2014
@@ -40,6 +40,17 @@ public class ExprNodeConstantDesc extend
   // If this constant was created while doing constant folding, foldedFromCol holds the name
of
   // original column from which it was folded.
   private transient String foldedFromCol;
+  // string representation of folding constant.
+  private transient String foldedFromVal;
+
+  public ExprNodeConstantDesc setFoldedFromVal(String foldedFromVal) {
+    this.foldedFromVal = foldedFromVal;
+    return this;
+  }
+
+  public String getFoldedFromVal() {
+    return foldedFromVal;
+  }
 
   public String getFoldedFromCol() {
     return foldedFromCol;

Modified: hive/trunk/ql/src/test/queries/clientpositive/constprog_type.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/constprog_type.q?rev=1633723&r1=1633722&r2=1633723&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/constprog_type.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/constprog_type.q Wed Oct 22 22:21:44 2014
@@ -12,3 +12,28 @@ SELECT cast('2013-11-17' as date), cast(
        FROM src tablesample (1 rows);
 
 SELECT * FROM dest1;
+
+SELECT key, value FROM src WHERE key = cast(86 as double);
+
+CREATE TABLE primitives1 (
+  id INT  ,
+  bool_col BOOLEAN  ,
+  tinyint_col TINYINT  ,
+  smallint_col SMALLINT  ,
+  int_col INT  ,
+  bigint_col BIGINT  ,
+  float_col FLOAT  ,
+  double_col DOUBLE  ,
+  date_string_col STRING  ,
+  string_col STRING  ,
+  timestamp_col TIMESTAMP  )
+ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY ','
+  ESCAPED BY '\\'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt'
+OVERWRITE INTO TABLE primitives1 ;
+
+
+select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col from
primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col
= cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double)
and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and  double_col =
cast (0.0 as float);

Modified: hive/trunk/ql/src/test/results/clientpositive/constprog_type.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/constprog_type.q.out?rev=1633723&r1=1633722&r2=1633723&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/constprog_type.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/constprog_type.q.out Wed Oct 22 22:21:44
2014
@@ -122,3 +122,69 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@dest1
 #### A masked pattern was here ####
 2013-11-17	2011-04-29 20:46:56.4485
+PREHOOK: query: SELECT key, value FROM src WHERE key = cast(86 as double)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT key, value FROM src WHERE key = cast(86 as double)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+86	val_86
+PREHOOK: query: CREATE TABLE primitives1 (
+  id INT  ,
+  bool_col BOOLEAN  ,
+  tinyint_col TINYINT  ,
+  smallint_col SMALLINT  ,
+  int_col INT  ,
+  bigint_col BIGINT  ,
+  float_col FLOAT  ,
+  double_col DOUBLE  ,
+  date_string_col STRING  ,
+  string_col STRING  ,
+  timestamp_col TIMESTAMP  )
+ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY ','
+  ESCAPED BY '\\'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@primitives1
+POSTHOOK: query: CREATE TABLE primitives1 (
+  id INT  ,
+  bool_col BOOLEAN  ,
+  tinyint_col TINYINT  ,
+  smallint_col SMALLINT  ,
+  int_col INT  ,
+  bigint_col BIGINT  ,
+  float_col FLOAT  ,
+  double_col DOUBLE  ,
+  date_string_col STRING  ,
+  string_col STRING  ,
+  timestamp_col TIMESTAMP  )
+ROW FORMAT DELIMITED
+  FIELDS TERMINATED BY ','
+  ESCAPED BY '\\'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@primitives1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt'
+OVERWRITE INTO TABLE primitives1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@primitives1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/types/primitives/090101.txt'
+OVERWRITE INTO TABLE primitives1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@primitives1
+PREHOOK: query: select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col
from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col
= cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double)
and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and  double_col =
cast (0.0 as float)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@primitives1
+#### A masked pattern was here ####
+POSTHOOK: query: select id,bool_col,tinyint_col,smallint_col,int_col,bigint_col,float_col,double_col
from primitives1 where id = cast (0 as float) and bool_col = cast('true' as boolean) and tinyint_col
= cast(0 as double) and smallint_col = cast(0 as bigint) and int_col = cast (0 as double)
and bigint_col = cast(0 as tinyint) and float_col = cast(0.0 as string) and  double_col =
cast (0.0 as float)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@primitives1
+#### A masked pattern was here ####
+0	true	0	0	0	0	0.0	0.0



Mime
View raw message