hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sd...@apache.org
Subject svn commit: r1163455 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/ test/queries/clientnegative/ test/results/clientnegative/
Date Wed, 31 Aug 2011 00:17:24 GMT
Author: sdong
Date: Wed Aug 31 00:17:24 2011
New Revision: 1163455

URL: http://svn.apache.org/viewvc?rev=1163455&view=rev
Log:
HIVE-2378. Warn user that precision is lost when bigint is implicitly cast to double. (Kevin
Wilfong via Siying Dong)

Added:
    hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q
    hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q
    hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out
    hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=1163455&r1=1163454&r2=1163455&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Wed Aug 31 00:17:24
2011
@@ -180,6 +180,12 @@ public enum ErrorMsg {
   INCOMPATIBLE_SCHEMA("The existing table is not compatible with the import spec. "),
   EXIM_FOR_NON_NATIVE("Export/Import cannot be done for a non-native table. "),
   INSERT_INTO_BUCKETIZED_TABLE("Bucketized tables do not support INSERT INTO:"),
+  NO_COMPARE_BIGINT_STRING("In strict mode, comparing bigints and strings is not allowed,
"
+      + "it may result in a loss of precision. "
+      + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"),
+  NO_COMPARE_BIGINT_DOUBLE("In strict mode, comparing bigints and doubles is not allowed,
"
+      + "it may result in a loss of precision. "
+      + "If you really want to perform the operation, set hive.mapred.mode=nonstrict"),
       ;
 
   private String mesg;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java?rev=1163455&r1=1163454&r2=1163455&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java Wed
Aug 31 00:17:24 2011
@@ -22,13 +22,22 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.ErrorMsg;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 
 /**
@@ -39,9 +48,12 @@ public class ExprNodeGenericFuncDesc ext
 
   private static final long serialVersionUID = 1L;
 
+  private static final Log LOG = LogFactory
+      .getLog(ExprNodeGenericFuncDesc.class.getName());
+
   /**
    * In case genericUDF is Serializable, we will serialize the object.
-   * 
+   *
    * In case genericUDF does not implement Serializable, Java will remember the
    * class of genericUDF and creates a new instance when deserialized. This is
    * exactly what we want.
@@ -138,7 +150,7 @@ public class ExprNodeGenericFuncDesc ext
   /**
    * Create a exprNodeGenericFuncDesc based on the genericUDFClass and the
    * children parameters.
-   * 
+   *
    * @throws UDFArgumentException
    */
   public static ExprNodeGenericFuncDesc newInstance(GenericUDF genericUDF,
@@ -148,6 +160,36 @@ public class ExprNodeGenericFuncDesc ext
       childrenOIs[i] = children.get(i).getWritableObjectInspector();
     }
 
+    // Check if a bigint is implicitely cast to a double as part of a comparison
+    // Perform the check here instead of in GenericUDFBaseCompare to guarantee it is only
run once per operator
+    if (genericUDF instanceof GenericUDFBaseCompare && children.size() == 2) {
+
+      TypeInfo oiTypeInfo0 = children.get(0).getTypeInfo();
+      TypeInfo oiTypeInfo1 = children.get(1).getTypeInfo();
+
+      SessionState ss = SessionState.get();
+      Configuration conf = (ss != null) ? ss.getConf() : new Configuration();
+
+      LogHelper console = new LogHelper(LOG);
+
+      // For now, if a bigint is going to be cast to a double throw an error or warning
+      if ((oiTypeInfo0.equals(TypeInfoFactory.stringTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo))
||
+          (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.stringTypeInfo)))
{
+        if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict"))
{
+          throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_STRING.getMsg());
+        } else {
+          console.printError("WARNING: Comparing a bigint and a string may result in a loss
of precision.");
+        }
+      } else if ((oiTypeInfo0.equals(TypeInfoFactory.doubleTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.longTypeInfo))
||
+          (oiTypeInfo0.equals(TypeInfoFactory.longTypeInfo) && oiTypeInfo1.equals(TypeInfoFactory.doubleTypeInfo)))
{
+        if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict"))
{
+          throw new UDFArgumentException(ErrorMsg.NO_COMPARE_BIGINT_DOUBLE.getMsg());
+        } else {
+          console.printError("WARNING: Comparing a bigint and a double may result in a loss
of precision.");
+        }
+      }
+    }
+
     ObjectInspector oi = genericUDF.initialize(childrenOIs);
     return new ExprNodeGenericFuncDesc(TypeInfoUtils
         .getTypeInfoFromObjectInspector(oi), genericUDF, children);

Added: hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/compare_double_bigint.q Wed Aug 31 00:17:24
2011
@@ -0,0 +1,5 @@
+set hive.mapred.mode=strict;
+
+-- This should fail until we fix the issue with precision when casting a bigint to a double
+
+select * from src where cast(1 as bigint) = 1.0 limit 10;
\ No newline at end of file

Added: hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/compare_string_bigint.q Wed Aug 31 00:17:24
2011
@@ -0,0 +1,5 @@
+set hive.mapred.mode=strict;
+
+--This should fail until we fix the issue with precision when casting a bigint to a double
+
+select * from src where cast(1 as bigint) = '1' limit 10;
\ No newline at end of file

Added: hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/compare_double_bigint.q.out Wed Aug 31 00:17:24
2011
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 0:-1 Wrong arguments '1.0': In strict mode, comparing
bigints and doubles is not allowed, it may result in a loss of precision. If you really want
to perform the operation, set hive.mapred.mode=nonstrict

Added: hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out?rev=1163455&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/compare_string_bigint.q.out Wed Aug 31 00:17:24
2011
@@ -0,0 +1 @@
+FAILED: Error in semantic analysis: Line 0:-1 Wrong arguments ''1'': In strict mode, comparing
bigints and strings is not allowed, it may result in a loss of precision. If you really want
to perform the operation, set hive.mapred.mode=nonstrict



Mime
View raw message