hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ych...@apache.org
Subject hive git commit: HIVE-13065: Hive throws NPE when writing map type data to a HBase backed table (Yongzhi Chen, reviewed by Aihua Xu)
Date Thu, 18 Feb 2016 18:55:41 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 39285e5cc -> 9b8fbd486


HIVE-13065: Hive throws NPE when writing map type data to a HBase backed table (Yongzhi Chen,
reviewed by Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9b8fbd48
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9b8fbd48
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9b8fbd48

Branch: refs/heads/branch-1
Commit: 9b8fbd4866f28aa3174a41f461a07afc330a172a
Parents: 39285e5
Author: Yongzhi Chen <ychena@apache.org>
Authored: Tue Feb 16 16:12:28 2016 -0500
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Thu Feb 18 13:41:26 2016 -0500

----------------------------------------------------------------------
 .../hadoop/hive/hbase/HBaseRowSerializer.java   |   7 +-
 .../src/test/queries/positive/hbase_queries.q   |  26 ++++
 .../test/results/positive/hbase_queries.q.out   | 154 +++++++++++++++++++
 3 files changed, 185 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9b8fbd48/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
index 7c141fc..c6f3b0f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
@@ -271,8 +271,11 @@ public class HBaseRowSerializer {
               ss.write(sep);
             }
             serialize(entry.getKey(), koi, level+2, ss);
-            ss.write(keyValueSeparator);
-            serialize(entry.getValue(), voi, level+2, ss);
+
+            if ( entry.getValue() != null) {
+              ss.write(keyValueSeparator);
+              serialize(entry.getValue(), voi, level+2, ss);
+            }
           }
         }
         return true;

http://git-wip-us.apache.org/repos/asf/hive/blob/9b8fbd48/hbase-handler/src/test/queries/positive/hbase_queries.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_queries.q b/hbase-handler/src/test/queries/positive/hbase_queries.q
index b445c4b..49fa829 100644
--- a/hbase-handler/src/test/queries/positive/hbase_queries.q
+++ b/hbase-handler/src/test/queries/positive/hbase_queries.q
@@ -164,6 +164,30 @@ SELECT COUNT(*) FROM hbase_table_1_like;
 
 SHOW CREATE TABLE hbase_table_1_like;
 
+DROP TABLE IF EXISTS hbase_table_9;
+CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col");
+
+insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as
str from src limit 1;
+insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str
from src limit 1;
+insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from
src limit 1;
+insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as
str from src limit 1;
+insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data
, '1234' as str from src limit 1;
+select * from hbase_table_9;
+
+DROP TABLE IF EXISTS hbase_table_10;
+CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col");
+insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data
, null as str from src limit 1;
+insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234'
as str from src limit 1;
+insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from
src limit 1;
+insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null
as int)) as data , '1234' as str from src limit 1;
+insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data
, '1234' as str from src limit 1;
+select * from hbase_table_10;
+
+
 DROP TABLE hbase_table_1;
 DROP TABLE hbase_table_1_like;
 DROP TABLE hbase_table_2;
@@ -176,3 +200,5 @@ DROP TABLE hbase_table_7;
 DROP TABLE hbase_table_8;
 DROP TABLE empty_hbase_table;
 DROP TABLE empty_normal_table;
+DROP TABLE hbase_table_9;
+DROP TABLE hbase_table_10;

http://git-wip-us.apache.org/repos/asf/hive/blob/9b8fbd48/hbase-handler/src/test/results/positive/hbase_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index d044c7e..4b51969 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -918,6 +918,144 @@ WITH SERDEPROPERTIES (
 TBLPROPERTIES (
   'hbase.table.name'='hbase_table_0', 
 #### A masked pattern was here ####
+PREHOOK: query: DROP TABLE IF EXISTS hbase_table_9
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_9
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str
string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str
string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as
data , null as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as
data , null as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 2 as id, map('efgh', null) as data
, '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 2 as id, map('efgh', null) as data
, '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 3 as id, map('hij', '') as data ,
'1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 3 as id, map('hij', '') as data ,
'1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data
, '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as
data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2',
'avalue') as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2',
'avalue') as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: select * from hbase_table_9
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_9
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_9
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_9
+#### A masked pattern was here ####
+1	{"abcd":null}	NULL
+2	{"efgh":null}	1234
+3	{"hij":""}	1234
+4	{"klm":"avalue"}	1234
+5	{"key1":null,"key2":"avalue"}	1234
+PREHOOK: query: DROP TABLE IF EXISTS hbase_table_10
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_10
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as
int)) as data , null as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null
as int)) as data , null as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 2 as id, map(20, cast(null as int))
as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 2 as id, map(20, cast(null as int))
as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234'
as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234'
as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 4 as id, map(40, cast(null as int),
45, cast(null as int)) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 4 as id, map(40, cast(null as int),
45, cast(null as int)) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 5 as id, map(50,cast(null as int),
55, 58) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 5 as id, map(50,cast(null as int),
55, 58) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: select * from hbase_table_10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_10
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_10
+#### A masked pattern was here ####
+1	{10:null}	NULL
+2	{20:null}	1234
+3	{30:31}	1234
+4	{40:null,45:null}	1234
+5	{50:null,55:58}	1234
 PREHOOK: query: DROP TABLE hbase_table_1
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@hbase_table_1
@@ -1014,3 +1152,19 @@ POSTHOOK: query: DROP TABLE empty_normal_table
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@empty_normal_table
 POSTHOOK: Output: default@empty_normal_table
+PREHOOK: query: DROP TABLE hbase_table_9
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_9
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: DROP TABLE hbase_table_9
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_9
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: DROP TABLE hbase_table_10
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_10
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: DROP TABLE hbase_table_10
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_10
+POSTHOOK: Output: default@hbase_table_10


Mime
View raw message