hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1631612 [1/3] - in /hive/trunk: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/ ql/src/test/results/clientpositive/tez/
Date Tue, 14 Oct 2014 04:52:13 GMT
Author: gunther
Date: Tue Oct 14 04:52:12 2014
New Revision: 1631612

URL: http://svn.apache.org/r1631612
Log:
HIVE-8434: Vectorization logic using wrong values for DATE and TIMESTAMP partitioning columns
in vectorized row batches... (Matt McCline via Gunther Hagleitner)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q
    hive/trunk/ql/src/test/results/clientpositive/tez/vector_partitioned_date_time.q.out
    hive/trunk/ql/src/test/results/clientpositive/vector_partitioned_date_time.q.out
Modified:
    hive/trunk/itests/src/test/resources/testconfiguration.properties
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java

Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1631612&r1=1631611&r2=1631612&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Tue Oct 14 04:52:12
2014
@@ -169,6 +169,7 @@ minitez.query.files.shared=alter_merge_2
   vector_mapjoin_reduce.q,\
   vector_non_string_partition.q,\
   vector_orderby_5.q,\
+  vector_partitioned_date_time.q,\
   vector_string_concat.q,\
   vector_varchar_4.q,\
   vector_varchar_simple.q,\

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java?rev=1631612&r1=1631611&r2=1631612&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
Tue Oct 14 04:52:12 2014
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.serde2.Col
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -487,7 +488,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill(((Date) value).getTime());
+            lcv.fill(DateWritable.dateToDays((Date) value));
             lcv.isNull[0] = false;
           }          
         }
@@ -500,7 +501,7 @@ public class VectorizedRowBatchCtx {
             lcv.isNull[0] = true;
             lcv.isRepeating = true;
           } else { 
-            lcv.fill((long)(((Timestamp) value).getTime()));
+            lcv.fill(TimestampUtils.getTimeNanoSec((Timestamp) value));
             lcv.isNull[0] = false;
           }
         }

Added: hive/trunk/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q?rev=1631612&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/vector_partitioned_date_time.q Tue Oct 14
04:52:12 2014
@@ -0,0 +1,127 @@
+set hive.fetch.task.conversion=minimal;
+
+
+-- Check if vectorization code is handling partitioning on DATE and the other data types.
+
+
+CREATE TABLE flights_tiny (
+  origin_city_name STRING,
+  dest_city_name STRING,
+  fl_date DATE,
+  arr_delay FLOAT,
+  fl_num INT
+);
+
+LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt.1' OVERWRITE INTO TABLE flights_tiny;
+
+CREATE TABLE flights_tiny_orc STORED AS ORC AS
+SELECT origin_city_name, dest_city_name, fl_date, to_utc_timestamp(fl_date, 'America/Los_Angeles')
as fl_time, arr_delay, fl_num
+FROM flights_tiny;
+
+SELECT * FROM flights_tiny_orc;
+
+SET hive.vectorized.execution.enabled=false;
+
+select * from flights_tiny_orc sort by fl_num, fl_date limit 25;
+
+select fl_date, count(*) from flights_tiny_orc group by fl_date;
+
+SET hive.vectorized.execution.enabled=true;
+
+explain
+select * from flights_tiny_orc sort by fl_num, fl_date limit 25;
+
+select * from flights_tiny_orc sort by fl_num, fl_date limit 25;
+
+explain
+select fl_date, count(*) from flights_tiny_orc group by fl_date;
+
+select fl_date, count(*) from flights_tiny_orc group by fl_date;
+
+
+SET hive.vectorized.execution.enabled=false;
+
+CREATE TABLE flights_tiny_orc_partitioned_date (
+  origin_city_name STRING,
+  dest_city_name STRING,
+  fl_time TIMESTAMP,
+  arr_delay FLOAT,
+  fl_num INT
+)
+PARTITIONED BY (fl_date DATE)
+STORED AS ORC;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT INTO TABLE flights_tiny_orc_partitioned_date
+PARTITION (fl_date)
+SELECT  origin_city_name, dest_city_name, fl_time, arr_delay, fl_num, fl_date
+FROM flights_tiny_orc;
+
+
+select * from flights_tiny_orc_partitioned_date;
+
+select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25;
+
+select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date;
+
+SET hive.vectorized.execution.enabled=true;
+
+explain
+select * from flights_tiny_orc_partitioned_date;
+
+select * from flights_tiny_orc_partitioned_date;
+
+explain
+select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25;
+
+select * from flights_tiny_orc_partitioned_date sort by fl_num, fl_date limit 25;
+
+explain
+select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date;
+
+select fl_date, count(*) from flights_tiny_orc_partitioned_date group by fl_date;
+
+
+SET hive.vectorized.execution.enabled=false;
+
+CREATE TABLE flights_tiny_orc_partitioned_timestamp (
+  origin_city_name STRING,
+  dest_city_name STRING,
+  fl_date DATE,
+  arr_delay FLOAT,
+  fl_num INT
+)
+PARTITIONED BY (fl_time TIMESTAMP)
+STORED AS ORC;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT INTO TABLE flights_tiny_orc_partitioned_timestamp
+PARTITION (fl_time)
+SELECT  origin_city_name, dest_city_name, fl_date, arr_delay, fl_num, fl_time
+FROM flights_tiny_orc;
+
+
+select * from flights_tiny_orc_partitioned_timestamp;
+
+select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25;
+
+select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time;
+
+SET hive.vectorized.execution.enabled=true;
+
+explain
+select * from flights_tiny_orc_partitioned_timestamp;
+
+select * from flights_tiny_orc_partitioned_timestamp;
+
+explain
+select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25;
+
+select * from flights_tiny_orc_partitioned_timestamp sort by fl_num, fl_time limit 25;
+
+explain
+select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time;
+
+select fl_time, count(*) from flights_tiny_orc_partitioned_timestamp group by fl_time;
\ No newline at end of file



Mime
View raw message