hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject hive git commit: HIVE-12012: select query on json table with map containing numeric values fails (Jason Dere, reviewed by Sushanth Sowmyan)
Date Fri, 09 Oct 2015 17:23:25 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 91fe1e161 -> 810c0f27a


HIVE-12012: select query on json table with map containing numeric values fails (Jason Dere,
reviewed by Sushanth Sowmyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/810c0f27
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/810c0f27
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/810c0f27

Branch: refs/heads/branch-1
Commit: 810c0f27ab7dbdbb8de8c539d4627c067bd2bc04
Parents: 91fe1e1
Author: Jason Dere <jdere@hortonworks.com>
Authored: Fri Oct 9 09:59:35 2015 -0700
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Fri Oct 9 10:02:02 2015 -0700

----------------------------------------------------------------------
 .../apache/hive/hcatalog/data/JsonSerDe.java    |   8 +-
 .../hive/hcatalog/data/TestJsonSerDe.java       |  36 ++++++
 .../test/queries/clientpositive/json_serde1.q   |  36 ++++++
 .../results/clientpositive/json_serde1.q.out    | 113 +++++++++++++++++++
 4 files changed, 186 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/810c0f27/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
index 9b325b6..1b47b28 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
@@ -349,13 +349,7 @@ public class JsonSerDe implements SerDe {
       HCatFieldSchema valueSchema = hcatFieldSchema.getMapValueSchema().get(0);
       while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
         Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), hcatFieldSchema.getMapKeyTypeInfo());
-        Object v;
-        if (valueSchema.getType() == HCatFieldSchema.Type.STRUCT) {
-          v = extractCurrentField(p, valueSchema, false);
-        } else {
-          v = extractCurrentField(p, valueSchema, true);
-        }
-
+        Object v = extractCurrentField(p, valueSchema, false);
         map.put(k, v);
       }
       val = map;

http://git-wip-us.apache.org/repos/asf/hive/blob/810c0f27/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
index 618f39b..5ececb5 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
@@ -307,4 +307,40 @@ public class TestJsonSerDe extends TestCase {
     assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2));
 
   }
+
+  private static HashMap<String, Integer> createHashMapStringInteger(Object...vals)
{
+    assertTrue(vals.length % 2 == 0);
+    HashMap<String, Integer> retval = new HashMap<String, Integer>();
+    for (int idx = 0; idx < vals.length; idx += 2) {
+      retval.put((String) vals[idx], (Integer) vals[idx+1]);
+    }
+    return retval;
+  }
+
+  public void testMapValues() throws Exception {
+    Configuration conf = new Configuration();
+    Properties props = new Properties();
+
+    props.put(serdeConstants.LIST_COLUMNS, "a,b");
+    props.put(serdeConstants.LIST_COLUMN_TYPES, "array<string>,map<string,int>");
+    JsonSerDe rjsd = new JsonSerDe();
+    SerDeUtils.initializeSerDe(rjsd, conf, props, null);
+
+    Text text1 = new Text("{ \"a\":[\"aaa\"],\"b\":{\"bbb\":1}} ");
+    Text text2 = new Text("{\"a\":[\"yyy\"],\"b\":{\"zzz\":123}}");
+    Text text3 = new Text("{\"a\":[\"a\"],\"b\":{\"x\":11, \"y\": 22, \"z\": null}}");
+
+    HCatRecord expected1 = new DefaultHCatRecord(Arrays.<Object>asList(
+        Arrays.<String>asList("aaa"),
+        createHashMapStringInteger("bbb", 1)));
+    HCatRecord expected2 = new DefaultHCatRecord(Arrays.<Object>asList(
+        Arrays.<String>asList("yyy"),
+        createHashMapStringInteger("zzz", 123)));
+    HCatRecord expected3 = new DefaultHCatRecord(Arrays.<Object>asList(
+        Arrays.<String>asList("a"),
+        createHashMapStringInteger("x", 11, "y", 22, "z", null)));
+
+    assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text1), expected1));
+    assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/810c0f27/ql/src/test/queries/clientpositive/json_serde1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/json_serde1.q b/ql/src/test/queries/clientpositive/json_serde1.q
new file mode 100644
index 0000000..85f5af2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/json_serde1.q
@@ -0,0 +1,36 @@
+
+add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar;
+
+drop table if exists json_serde1_1;
+drop table if exists json_serde1_2;
+
+create table json_serde1_1 (a array<string>,b map<string,int>)
+  row format serde 'org.apache.hive.hcatalog.data.JsonSerDe';
+
+insert into table json_serde1_1
+  select array('aaa'),map('aaa',1) from src limit 2;
+
+select * from json_serde1_1;
+
+create table json_serde1_2 (
+  a array<int>,
+  b map<int,date>,
+  c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string,
c5_2:int>>
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe';
+
+insert into table json_serde1_2
+  select
+    array(3, 2, 1),
+    map(1, date '2001-01-01', 2, null),
+    named_struct(
+      'c1', 123456,
+      'c2', 'hello',
+      'c3', array('aa', 'bb', 'cc'),
+      'c4', map('abc', 123, 'xyz', 456),
+      'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+  from src limit 2;
+
+select * from json_serde1_2;
+
+drop table json_serde1_1;
+drop table json_serde1_2;

http://git-wip-us.apache.org/repos/asf/hive/blob/810c0f27/ql/src/test/results/clientpositive/json_serde1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/json_serde1.q.out b/ql/src/test/results/clientpositive/json_serde1.q.out
new file mode 100644
index 0000000..6235aff
--- /dev/null
+++ b/ql/src/test/results/clientpositive/json_serde1.q.out
@@ -0,0 +1,113 @@
+PREHOOK: query: drop table if exists json_serde1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists json_serde1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists json_serde1_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists json_serde1_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table json_serde1_1 (a array<string>,b map<string,int>)
+  row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: create table json_serde1_1 (a array<string>,b map<string,int>)
+  row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde1_1
+PREHOOK: query: insert into table json_serde1_1
+  select array('aaa'),map('aaa',1) from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: insert into table json_serde1_1
+  select array('aaa'),map('aaa',1) from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@json_serde1_1
+POSTHOOK: Lineage: json_serde1_1.a EXPRESSION []
+POSTHOOK: Lineage: json_serde1_1.b EXPRESSION []
+PREHOOK: query: select * from json_serde1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@json_serde1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from json_serde1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@json_serde1_1
+#### A masked pattern was here ####
+["aaa"]	{"aaa":1}
+["aaa"]	{"aaa":1}
+PREHOOK: query: create table json_serde1_2 (
+  a array<int>,
+  b map<int,date>,
+  c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string,
c5_2:int>>
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: create table json_serde1_2 (
+  a array<int>,
+  b map<int,date>,
+  c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string,
c5_2:int>>
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde1_2
+PREHOOK: query: insert into table json_serde1_2
+  select
+    array(3, 2, 1),
+    map(1, date '2001-01-01', 2, null),
+    named_struct(
+      'c1', 123456,
+      'c2', 'hello',
+      'c3', array('aa', 'bb', 'cc'),
+      'c4', map('abc', 123, 'xyz', 456),
+      'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+  from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: insert into table json_serde1_2
+  select
+    array(3, 2, 1),
+    map(1, date '2001-01-01', 2, null),
+    named_struct(
+      'c1', 123456,
+      'c2', 'hello',
+      'c3', array('aa', 'bb', 'cc'),
+      'c4', map('abc', 123, 'xyz', 456),
+      'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+  from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@json_serde1_2
+POSTHOOK: Lineage: json_serde1_2.a EXPRESSION []
+POSTHOOK: Lineage: json_serde1_2.b EXPRESSION []
+POSTHOOK: Lineage: json_serde1_2.c EXPRESSION []
+PREHOOK: query: select * from json_serde1_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@json_serde1_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from json_serde1_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@json_serde1_2
+#### A masked pattern was here ####
+[3,2,1]	{1:"2001-01-01",2:null}	{"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"xyz":456,"abc":123},"c5":{"c5_1":"bye","c5_2":88}}
+[3,2,1]	{1:"2001-01-01",2:null}	{"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"xyz":456,"abc":123},"c5":{"c5_1":"bye","c5_2":88}}
+PREHOOK: query: drop table json_serde1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde1_1
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: drop table json_serde1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde1_1
+POSTHOOK: Output: default@json_serde1_1
+PREHOOK: query: drop table json_serde1_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde1_2
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: drop table json_serde1_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde1_2
+POSTHOOK: Output: default@json_serde1_2


Mime
View raw message