hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1619024 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/io/orc/ java/org/apache/hadoop/hive/ql/io/sarg/ test/org/apache/hadoop/hive/ql/io/sarg/ test/queries/clientpositive/ test/results/clientpositive/
Date Wed, 20 Aug 2014 02:34:33 GMT
Author: navis
Date: Wed Aug 20 02:34:32 2014
New Revision: 1619024

URL: http://svn.apache.org/r1619024
Log:
HIVE-7771 : ORC PPD fails for some decimal predicates (Prasanth J, reviwed by Daniel Dai)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
    hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q
    hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Wed Aug
20 02:34:32 2014
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hive.con
 
 import java.io.EOFException;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.sql.Timestamp;
@@ -2411,6 +2412,9 @@ class RecordReaderImpl implements Record
 
   private static Object getBaseObjectForComparison(Object predObj, Object statsObj) {
     if (predObj != null) {
+      if (predObj instanceof ExprNodeConstantDesc) {
+        predObj = ((ExprNodeConstantDesc) predObj).getValue();
+      }
       // following are implicitly convertible
       if (statsObj instanceof Long) {
         if (predObj instanceof Double) {
@@ -2429,10 +2433,6 @@ class RecordReaderImpl implements Record
           return Double.valueOf(predObj.toString());
         }
       } else if (statsObj instanceof String) {
-        // Ex: where d = date '1970-02-01' will be ExprNodeConstantDesc
-        if (predObj instanceof ExprNodeConstantDesc) {
-          return ((ExprNodeConstantDesc) predObj).getValue().toString();
-        }
         return predObj.toString();
       } else if (statsObj instanceof HiveDecimal) {
         if (predObj instanceof Long) {
@@ -2441,6 +2441,8 @@ class RecordReaderImpl implements Record
           return HiveDecimal.create(predObj.toString());
         } else if (predObj instanceof String) {
           return HiveDecimal.create(predObj.toString());
+        } else if (predObj instanceof BigDecimal) {
+          return HiveDecimal.create((BigDecimal)predObj);
         }
       }
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java Wed Aug
20 02:34:32 2014
@@ -18,14 +18,6 @@
 
 package org.apache.hadoop.hive.ql.io.sarg;
 
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
@@ -57,6 +49,15 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 
+import java.math.BigDecimal;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * The implementation of SearchArguments.
  */
@@ -947,7 +948,8 @@ final class SearchArgumentImpl implement
           literal instanceof Long ||
           literal instanceof Double ||
           literal instanceof DateWritable ||
-          literal instanceof HiveDecimal) {
+          literal instanceof HiveDecimal ||
+          literal instanceof BigDecimal) {
         return literal;
       } else if (literal instanceof HiveChar ||
           literal instanceof HiveVarchar) {
@@ -979,7 +981,8 @@ final class SearchArgumentImpl implement
         return PredicateLeaf.Type.FLOAT;
       } else if (literal instanceof DateWritable) {
         return PredicateLeaf.Type.DATE;
-      } else if (literal instanceof HiveDecimal) {
+      } else if (literal instanceof HiveDecimal ||
+          literal instanceof BigDecimal) {
         return PredicateLeaf.Type.DECIMAL;
       }
       throw new IllegalArgumentException("Unknown type for literal " + literal);

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java Wed
Aug 20 02:34:32 2014
@@ -18,15 +18,7 @@
 
 package org.apache.hadoop.hive.ql.io.sarg;
 
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertTrue;
-
-import java.beans.XMLDecoder;
-import java.io.ByteArrayInputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.List;
-import java.util.Set;
-
+import com.google.common.collect.Sets;
 import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.common.type.HiveVarchar;
@@ -37,7 +29,15 @@ import org.apache.hadoop.hive.ql.plan.Ex
 import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.junit.Test;
 
-import com.google.common.collect.Sets;
+import java.beans.XMLDecoder;
+import java.io.ByteArrayInputStream;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Set;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
 
 /**
  * These test the SARG implementation.
@@ -2847,4 +2847,36 @@ public class TestSearchArgumentImpl {
         "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
         "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString());
   }
+
+  @Test
+  public void testBuilderComplexTypes2() throws Exception {
+    SearchArgument sarg =
+        SearchArgument.FACTORY.newBuilder()
+            .startAnd()
+            .lessThan("x", new DateWritable(10))
+            .lessThanEquals("y", new HiveChar("hi", 10))
+            .equals("z", new BigDecimal("1.0"))
+            .end()
+            .build();
+    assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" +
+        "leaf-1 = (LESS_THAN_EQUALS y hi)\n" +
+        "leaf-2 = (EQUALS z 1.0)\n" +
+        "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString());
+
+    sarg = SearchArgument.FACTORY.newBuilder()
+        .startNot()
+        .startOr()
+        .isNull("x")
+        .between("y", new BigDecimal(10), 20.0)
+        .in("z", (byte)1, (short)2, (int)3)
+        .nullSafeEquals("a", new HiveVarchar("stinger", 100))
+        .end()
+        .end()
+        .build();
+    assertEquals("leaf-0 = (IS_NULL x)\n" +
+        "leaf-1 = (BETWEEN y 10 20.0)\n" +
+        "leaf-2 = (IN z 1 2 3)\n" +
+        "leaf-3 = (NULL_SAFE_EQUALS a stinger)\n" +
+        "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toString());
+  }
 }

Modified: hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_decimal.q Wed Aug 20 02:34:32 2014
@@ -85,6 +85,18 @@ set hive.optimize.index.filter=true;
 select sum(hash(*)) from newtypesorc where d<=cast('11.22' as float);
 
 set hive.optimize.index.filter=false;
+select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal);
+
+set hive.optimize.index.filter=true;
+select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal);
+
+set hive.optimize.index.filter=false;
+select sum(hash(*)) from newtypesorc where d<=11.22BD;
+
+set hive.optimize.index.filter=true;
+select sum(hash(*)) from newtypesorc where d<=11.22BD;
+
+set hive.optimize.index.filter=false;
 select sum(hash(*)) from newtypesorc where d<=12;
 
 set hive.optimize.index.filter=true;

Modified: hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out?rev=1619024&r1=1619023&r2=1619024&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/orc_ppd_decimal.q.out Wed Aug 20 02:34:32
2014
@@ -254,6 +254,42 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@newtypesorc
 #### A masked pattern was here ####
 81475875500
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+-252951929000
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=cast('11.22' as decimal)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+-252951929000
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+81475875500
+PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+PREHOOK: type: QUERY
+PREHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<=11.22BD
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@newtypesorc
+#### A masked pattern was here ####
+81475875500
 PREHOOK: query: select sum(hash(*)) from newtypesorc where d<=12
 PREHOOK: type: QUERY
 PREHOOK: Input: default@newtypesorc



Mime
View raw message