hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1347212 - in /hive/trunk: ql/src/test/results/clientpositive/ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/
Date Thu, 07 Jun 2012 00:04:01 GMT
Author: hashutosh
Date: Thu Jun  7 00:04:00 2012
New Revision: 1347212

URL: http://svn.apache.org/viewvc?rev=1347212&view=rev
Log:
HIVE-2941: Hive should expand nested structs when setting the table schema from thrift structs
(Travis Crawford via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
    hive/trunk/ql/src/test/results/clientpositive/input17.q.out
    hive/trunk/ql/src/test/results/clientpositive/input5.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
    hive/trunk/ql/src/test/results/clientpositive/inputddl8.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java

Modified: hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out Thu Jun  7 00:04:00
2012
@@ -96,7 +96,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src_thrift
 POSTHOOK: Output: default@dest1
 POSTHOOK: Lineage: dest1.key EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>, comment:from deserializer),
]
+POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<struct<myint:int,mystring:string,underscore_int:int>>, comment:from
deserializer), ]
 PREHOOK: query: SELECT DEST1.* FROM Dest1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@dest1
@@ -106,7 +106,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@dest1
 #### A masked pattern was here ####
 POSTHOOK: Lineage: dest1.key EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>, comment:from deserializer),
]
+POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<struct<myint:int,mystring:string,underscore_int:int>>, comment:from
deserializer), ]
 2	1
 4	8
 6	27

Modified: hive/trunk/ql/src/test/results/clientpositive/input17.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input17.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input17.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input17.q.out Thu Jun  7 00:04:00 2012
@@ -121,8 +121,8 @@ INSERT OVERWRITE TABLE dest1 SELECT tmap
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src_thrift
 POSTHOOK: Output: default@dest1
-POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
 PREHOOK: query: SELECT dest1.* FROM dest1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@dest1
@@ -131,8 +131,8 @@ POSTHOOK: query: SELECT dest1.* FROM des
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@dest1
 #### A masked pattern was here ####
-POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:aint, type:int,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
 NULL	null
 -1461153966	{"myint":49,"mystring":"343","underscore_int":7}
 -1952710705	{"myint":25,"mystring":"125","underscore_int":5}

Modified: hive/trunk/ql/src/test/results/clientpositive/input5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input5.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input5.q.out Thu Jun  7 00:04:00 2012
@@ -114,8 +114,8 @@ INSERT OVERWRITE TABLE dest1 SELECT tmap
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src_thrift
 POSTHOOK: Output: default@dest1
-POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
 PREHOOK: query: SELECT dest1.* FROM dest1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@dest1
@@ -124,8 +124,8 @@ POSTHOOK: query: SELECT dest1.* FROM des
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@dest1
 #### A masked pattern was here ####
-POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.key SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.value SCRIPT [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), (src_thrift)src_thrift.FieldSchema(name:lintstring, type:array<struct<myint:int,mystring:string,underscore_int:int>>,
comment:from deserializer), ]
 [0,0,0]	[{"myint":0,"mystring":"0","underscore_int":0}]
 [1,2,3]	[{"myint":1,"mystring":"1","underscore_int":1}]
 [2,4,6]	[{"myint":4,"mystring":"8","underscore_int":2}]

Modified: hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out Thu Jun  7 00:04:00
2012
@@ -95,7 +95,7 @@ POSTHOOK: Input: default@src_thrift
 POSTHOOK: Output: default@dest1
 POSTHOOK: Lineage: dest1.key EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), ]
 POSTHOOK: Lineage: dest1.mapvalue EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:mstringstring,
type:map<string,string>, comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>, comment:from deserializer),
]
+POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<struct<myint:int,mystring:string,underscore_int:int>>, comment:from
deserializer), ]
 PREHOOK: query: SELECT dest1.* FROM dest1
 PREHOOK: type: QUERY
 PREHOOK: Input: default@dest1
@@ -106,7 +106,7 @@ POSTHOOK: Input: default@dest1
 #### A masked pattern was here ####
 POSTHOOK: Lineage: dest1.key EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>,
comment:from deserializer), ]
 POSTHOOK: Lineage: dest1.mapvalue EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:mstringstring,
type:map<string,string>, comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>, comment:from deserializer),
]
+POSTHOOK: Lineage: dest1.value EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<struct<myint:int,mystring:string,underscore_int:int>>, comment:from
deserializer), ]
 0	0	NULL
 2	1	NULL
 4	8	value_2

Modified: hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out Thu Jun  7 00:04:00
2012
@@ -98,7 +98,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src_thrift
 POSTHOOK: Output: default@dest1
 POSTHOOK: Lineage: dest1.lint_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lint,
type:array<int>, comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.lintstring_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>, comment:from deserializer),
]
+POSTHOOK: Lineage: dest1.lintstring_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<struct<myint:int,mystring:string,underscore_int:int>>, comment:from
deserializer), ]
 POSTHOOK: Lineage: dest1.mstringstring_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:mstringstring,
type:map<string,string>, comment:from deserializer), ]
 PREHOOK: query: SELECT dest1.* FROM dest1
 PREHOOK: type: QUERY
@@ -109,7 +109,7 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@dest1
 #### A masked pattern was here ####
 POSTHOOK: Lineage: dest1.lint_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lint,
type:array<int>, comment:from deserializer), ]
-POSTHOOK: Lineage: dest1.lintstring_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<org.apache.hadoop.hive.serde2.thrift.test.IntString>, comment:from deserializer),
]
+POSTHOOK: Lineage: dest1.lintstring_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:lintstring,
type:array<struct<myint:int,mystring:string,underscore_int:int>>, comment:from
deserializer), ]
 POSTHOOK: Lineage: dest1.mstringstring_size EXPRESSION [(src_thrift)src_thrift.FieldSchema(name:mstringstring,
type:map<string,string>, comment:from deserializer), ]
 3	1	1
 3	1	1

Modified: hive/trunk/ql/src/test/results/clientpositive/inputddl8.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/inputddl8.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/inputddl8.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/inputddl8.q.out Thu Jun  7 00:04:00 2012
@@ -23,7 +23,7 @@ aint	int	from deserializer
 astring	string	from deserializer
 lint	array<int>	from deserializer
 lstring	array<string>	from deserializer
-lintstring	array<org.apache.hadoop.hive.serde2.thrift.test.IntString>	from deserializer
+lintstring	array<struct<myint:int,mystring:string,underscore_int:int>>	from deserializer
 mstringstring	map<string,string>	from deserializer
 ds	string	
 country	string	

Modified: hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out Thu Jun  7 00:04:00 2012
@@ -6,7 +6,7 @@ aint	int	from deserializer
 astring	string	from deserializer
 lint	array<int>	from deserializer
 lstring	array<string>	from deserializer
-lintstring	array<org.apache.hadoop.hive.serde2.thrift.test.IntString>	from deserializer
+lintstring	array<struct<myint:int,mystring:string,underscore_int:int>>	from deserializer
 mstringstring	map<string,string>	from deserializer
 PREHOOK: query: EXPLAIN
 SELECT s1.aint, s2.lintstring

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java?rev=1347212&r1=1347211&r2=1347212&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java
Thu Jun  7 00:04:00 2012
@@ -76,7 +76,19 @@ public class ReflectionStructObjectInspe
   }
 
   public String getTypeName() {
-    return objectClass.getName();
+    StringBuilder sb = new StringBuilder("struct<");
+    boolean first = true;
+    for (StructField structField : getAllStructFieldRefs()) {
+      if (first) {
+        first = false;
+      } else {
+        sb.append(",");
+      }
+      sb.append(structField.getFieldName()).append(":")
+          .append(structField.getFieldObjectInspector().getTypeName());
+    }
+    sb.append(">");
+    return sb.toString();
   }
 
   /**



Mime
View raw message