hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-12475: Parquet schema evolution within array<struct<>> doesn't work (Mohammad Kamrul Islam, reviewed by Sergio Pena)
Date Wed, 02 Dec 2015 15:34:42 GMT
Repository: hive
Updated Branches:
  refs/heads/master aa61697b5 -> 232af9fae


HIVE-12475: Parquet schema evolution within array<struct<>> doesn't work (Mohammad
Kamrul Islam, reviewed by Sergio Pena)

Change-Id: I69a1dbf2122ddb60cf867c1820877aaa8207692d


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/232af9fa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/232af9fa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/232af9fa

Branch: refs/heads/master
Commit: 232af9faebce425a68c63ee09474a45848fa8fda
Parents: aa61697
Author: Mohammad Kamrul Islam <mislam77@yahoo.com>
Authored: Wed Dec 2 09:31:42 2015 -0600
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Wed Dec 2 09:33:42 2015 -0600

----------------------------------------------------------------------
 .../serde/ArrayWritableObjectInspector.java      |  8 ++++++--
 .../clientpositive/parquet_type_promotion.q      |  3 +++
 .../clientpositive/parquet_type_promotion.q.out  | 19 +++++++++++++++++++
 3 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/232af9fa/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java
index ae545b8..7873c99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java
@@ -136,10 +136,14 @@ public class ArrayWritableObjectInspector extends SettableStructObjectInspector
     if (data == null) {
       return null;
     }
-
     if (data instanceof ArrayWritable) {
       final ArrayWritable arr = (ArrayWritable) data;
-      return arr.get()[((StructFieldImpl) fieldRef).getIndex()];
+      final StructFieldImpl structField = (StructFieldImpl) fieldRef;
+      if (structField.getIndex() < arr.get().length) {
+        return arr.get()[structField.getIndex()];
+      } else {
+        return null;
+      }
     }
 
     //since setStructFieldData and create return a list, getStructFieldData should be able
to

http://git-wip-us.apache.org/repos/asf/hive/blob/232af9fa/ql/src/test/queries/clientpositive/parquet_type_promotion.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/parquet_type_promotion.q b/ql/src/test/queries/clientpositive/parquet_type_promotion.q
index 71abe3b..46a0f56 100644
--- a/ql/src/test/queries/clientpositive/parquet_type_promotion.q
+++ b/ql/src/test/queries/clientpositive/parquet_type_promotion.q
@@ -68,5 +68,8 @@ f2:int,f3:int>>) STORED AS PARQUET;
 INSERT INTO TABLE arrays_of_struct_to_map select array(named_struct("c1",1,"c2",2)), array(named_struct("f1",
 77,"f2",88,"f3",99)) FROM parquet_type_promotion LIMIT 1;
 SELECT * FROM arrays_of_struct_to_map;
+ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array<struct<c1:int,c2:int,c3:int>>,
locations2
+array<struct<f1:int,f2:int,f3:int>>);
+SELECT * FROM arrays_of_struct_to_map;
 ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 map<int,bigint>, locations2
map<bigint,int>);
 SELECT * FROM arrays_of_struct_to_map;

http://git-wip-us.apache.org/repos/asf/hive/blob/232af9fa/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/parquet_type_promotion.q.out b/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
index 15a29e3..dd08631 100644
--- a/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
+++ b/ql/src/test/results/clientpositive/parquet_type_promotion.q.out
@@ -213,6 +213,25 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@arrays_of_struct_to_map
 #### A masked pattern was here ####
 [{"c1":1,"c2":2}]	[{"f1":77,"f2":88,"f3":99}]
+PREHOOK: query: ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array<struct<c1:int,c2:int,c3:int>>,
locations2
+array<struct<f1:int,f2:int,f3:int>>)
+PREHOOK: type: ALTERTABLE_REPLACECOLS
+PREHOOK: Input: default@arrays_of_struct_to_map
+PREHOOK: Output: default@arrays_of_struct_to_map
+POSTHOOK: query: ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 array<struct<c1:int,c2:int,c3:int>>,
locations2
+array<struct<f1:int,f2:int,f3:int>>)
+POSTHOOK: type: ALTERTABLE_REPLACECOLS
+POSTHOOK: Input: default@arrays_of_struct_to_map
+POSTHOOK: Output: default@arrays_of_struct_to_map
+PREHOOK: query: SELECT * FROM arrays_of_struct_to_map
+PREHOOK: type: QUERY
+PREHOOK: Input: default@arrays_of_struct_to_map
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM arrays_of_struct_to_map
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@arrays_of_struct_to_map
+#### A masked pattern was here ####
+[{"c1":1,"c2":2,"c3":null}]	[{"f1":77,"f2":88,"f3":99}]
 PREHOOK: query: ALTER TABLE arrays_of_struct_to_map REPLACE COLUMNS (locations1 map<int,bigint>,
locations2 map<bigint,int>)
 PREHOOK: type: ALTERTABLE_REPLACECOLS
 PREHOOK: Input: default@arrays_of_struct_to_map


Mime
View raw message