hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r781121 - in /hadoop/hive/trunk: ./ data/files/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/ serde/if/test/ serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/...
Date Tue, 02 Jun 2009 18:51:21 GMT
Author: namit
Date: Tue Jun  2 18:51:20 2009
New Revision: 781121

URL: http://svn.apache.org/viewvc?rev=781121&view=rev
Log:
HIVE-495. Fix join of a table of ThriftSerDe with complex columns
(Zheng Shao via namit)


Added:
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/data/files/complex.seq
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
    hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
    hadoop/hive/trunk/serde/if/test/complex.thrift
    hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
    hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
    hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Jun  2 18:51:20 2009
@@ -162,7 +162,7 @@
     the expression string is empty. (Zheng Shao via athusoo)
 
     HIVE-410: Heartbeating for streaming jobs should not depend on stdout.
-    (Ashish Thusoo via njain)
+    (Ashish Thusoo via namit)
 
     HIVE-499. Fix for CAST operator on the same type. (Zheng Shao via athusoo)
 
@@ -181,7 +181,7 @@
     HIVE-500. Fix select from newly created table.
     (Yongqiang He via zshao)
 
-    HIVE-451. Fix ORDER BY xxx DESC (He Yongqiang via njain)
+    HIVE-451. Fix ORDER BY xxx DESC (He Yongqiang via namit)
 
     HIVE-467. Scratch data location should be on different filesystems for
     different types of intermediate data. (Joydeep Sen Sarma via rmurthy)
@@ -200,6 +200,9 @@
     HIVE-525. Fix Eclipse classpath by adding Hadoop test jar. 
     (Prasad Chakka via johan)
 
+    HIVE-495. Fix join of a table of ThriftSerDe with complex columns
+    (Zheng Shao via namit)
+
 Release 0.3.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/data/files/complex.seq
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/complex.seq?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Tue Jun
 2 18:51:20 2009
@@ -37,6 +37,7 @@
 import org.apache.hadoop.hive.ql.plan.joinDesc;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -44,6 +45,7 @@
 import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.io.Text;
@@ -161,8 +163,12 @@
     for (Byte alias : order) {
       int sz = map.get(alias).size();
       StructObjectInspector fldObjIns = (StructObjectInspector)((StructObjectInspector)inputObjInspector[alias.intValue()]).getStructFieldRef("VALUE").getFieldObjectInspector();
-      for (int i = 0; i < sz; i++)
-        structFieldObjectInspectors.add(fldObjIns.getAllStructFieldRefs().get(i).getFieldObjectInspector());
+      for (int i = 0; i < sz; i++) {
+        structFieldObjectInspectors.add(
+            ObjectInspectorUtils.getStandardObjectInspector(
+                fldObjIns.getAllStructFieldRefs().get(i).getFieldObjectInspector(),
+                ObjectInspectorCopyOption.KEEP));
+      }
     }
     
     joinOutputObjectInspector = ObjectInspectorFactory

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q?rev=781121&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_thrift.q Tue Jun  2 18:51:20
2009
@@ -0,0 +1,12 @@
+DESCRIBE src_thrift;
+
+EXPLAIN
+SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint;
+
+SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint;

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/case_sensitivity.q.out Tue Jun  2
18:51:20 2009
@@ -19,7 +19,7 @@
                     expr: lint
                     type: array<int>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
               Filter Operator
                 predicate:
                     expr: (0[0] > 0)
@@ -56,7 +56,7 @@
 Output: default/dest1
 query: SELECT DEST1.* FROM Dest1
 Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/256252840/400813971.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/154752222/89798452.10000
 2	1
 4	8
 6	27

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/describe_xpath.q.out Tue Jun  2 18:51:20
2009
@@ -14,6 +14,7 @@
 describe src_thrift.lintString.$elem$
 myint	int	from deserializer
 mystring	string	from deserializer
+underscore_int	int	from deserializer
 query: -- Describe a member of an element of a list
 describe src_thrift.lintString.$elem$.myint
 myint	int	from deserializer

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input17.q.out Tue Jun  2 18:51:20
2009
@@ -26,13 +26,13 @@
                     expr: lint
                     type: array<int>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
               Select Operator
                 expressions:
                       expr: (0 + 1[0])
                       type: int
                       expr: 2[0]
-                      type: struct<myint:int,mystring:string>
+                      type: struct<myint:int,mystring:string,underscore_int:int>
                 Transform Operator
                   command: /bin/cat
                   output info:
@@ -97,15 +97,15 @@
 Output: default/dest1
 query: SELECT dest1.* FROM dest1
 Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/10910343/1742990479.10000
-NULL	{"myint":null,"mystring":null}
--1461153966	{"myint":49,"mystring":"343"}
--1952710705	{"myint":25,"mystring":"125"}
--734328905	{"myint":16,"mystring":"64"}
--751827636	{"myint":4,"mystring":"8"}
-1244525196	{"myint":36,"mystring":"216"}
-1638581586	{"myint":64,"mystring":"512"}
-1712634731	{"myint":0,"mystring":"0"}
-336964422	{"myint":81,"mystring":"729"}
-465985201	{"myint":1,"mystring":"1"}
-477111225	{"myint":9,"mystring":"27"}
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/899544746/1236316343.10000
+NULL	{"myint":null,"mystring":null,"underscore_int":null}
+-1461153966	{"myint":49,"mystring":"343","underscore_int":7}
+-1952710705	{"myint":25,"mystring":"125","underscore_int":5}
+-734328905	{"myint":16,"mystring":"64","underscore_int":4}
+-751827636	{"myint":4,"mystring":"8","underscore_int":2}
+1244525196	{"myint":36,"mystring":"216","underscore_int":6}
+1638581586	{"myint":64,"mystring":"512","underscore_int":8}
+1712634731	{"myint":0,"mystring":"0","underscore_int":0}
+336964422	{"myint":81,"mystring":"729","underscore_int":9}
+465985201	{"myint":1,"mystring":"1","underscore_int":1}
+477111225	{"myint":9,"mystring":"27","underscore_int":3}

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input5.q.out Tue Jun  2 18:51:20
2009
@@ -24,13 +24,13 @@
                     expr: lint
                     type: array<int>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
               Select Operator
                 expressions:
                       expr: 0
                       type: array<int>
                       expr: 1
-                      type: array<struct<myint:int,mystring:string>>
+                      type: array<struct<myint:int,mystring:string,underscore_int:int>>
                 Transform Operator
                   command: /bin/cat
                   output info:
@@ -89,15 +89,15 @@
 Output: default/dest1
 query: SELECT dest1.* FROM dest1
 Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/248520134/578564891.10000
-[0,0,0]	[{"myint":0,"mystring":"0"}]
-[1,2,3]	[{"myint":1,"mystring":"1"}]
-[2,4,6]	[{"myint":4,"mystring":"8"}]
-[3,6,9]	[{"myint":9,"mystring":"27"}]
-[4,8,12]	[{"myint":16,"mystring":"64"}]
-[5,10,15]	[{"myint":25,"mystring":"125"}]
-[6,12,18]	[{"myint":36,"mystring":"216"}]
-[7,14,21]	[{"myint":49,"mystring":"343"}]
-[8,16,24]	[{"myint":64,"mystring":"512"}]
-[9,18,27]	[{"myint":81,"mystring":"729"}]
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/1082689233/217248273.10000
+[0,0,0]	[{"myint":0,"mystring":"0","underscore_int":0}]
+[1,2,3]	[{"myint":1,"mystring":"1","underscore_int":1}]
+[2,4,6]	[{"myint":4,"mystring":"8","underscore_int":2}]
+[3,6,9]	[{"myint":9,"mystring":"27","underscore_int":3}]
+[4,8,12]	[{"myint":16,"mystring":"64","underscore_int":4}]
+[5,10,15]	[{"myint":25,"mystring":"125","underscore_int":5}]
+[6,12,18]	[{"myint":36,"mystring":"216","underscore_int":6}]
+[7,14,21]	[{"myint":49,"mystring":"343","underscore_int":7}]
+[8,16,24]	[{"myint":64,"mystring":"512","underscore_int":8}]
+[9,18,27]	[{"myint":81,"mystring":"729","underscore_int":9}]
 null	null

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath.q.out Tue Jun  2
18:51:20 2009
@@ -19,7 +19,7 @@
                     expr: lint
                     type: array<int>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
                     expr: mstringstring
                     type: map<string,string>
               Select Operator
@@ -56,7 +56,7 @@
 Output: default/dest1
 query: SELECT dest1.* FROM dest1
 Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/403047878/395453459.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/600192558/110692382.10000
 0	0	NULL
 2	1	NULL
 4	8	value_2

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath2.q.out Tue Jun  2
18:51:20 2009
@@ -19,7 +19,7 @@
                     expr: lint
                     type: array<int>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
                     expr: mstringstring
                     type: map<string,string>
               Filter Operator
@@ -60,7 +60,7 @@
 Output: default/dest1
 query: SELECT dest1.* FROM dest1
 Input: default/dest1
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/627000844/1811724340.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/108328031/145804882.10000
 3	1	1
 3	1	1
 3	1	1

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath3.q.out Tue Jun  2
18:51:20 2009
@@ -18,7 +18,7 @@
                     expr: mstringstring
                     type: map<string,string>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
               Select Operator
                 expressions:
                       expr: 0['key_9']
@@ -40,7 +40,7 @@
 query: FROM src_thrift
 SELECT src_thrift.mstringstring['key_9'], src_thrift.lintstring.myint
 Input: default/src_thrift
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/535177843/787258899.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/47313672/383253534.10000
 NULL	[0]
 NULL	[1]
 NULL	[4]

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input_testxpath4.q.out Tue Jun  2
18:51:20 2009
@@ -21,7 +21,7 @@
                     expr: mstringstring
                     type: map<string,string>
                     expr: lintstring
-                    type: array<struct<myint:int,mystring:string>>
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
               Filter Operator
                 predicate:
                     expr: ((0['key_9'] is not null and 1.myint is not null) and 1 is not
null)
@@ -50,7 +50,7 @@
       OR lintstring.myint IS NOT NULL
       OR lintstring IS NOT NULL
 Input: default/src_thrift
-Output: /data/users/athusoo/commits/hive_trunk_ws8/ql/../build/ql/tmp/9314654/120165410.10000
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/115422682/57144029.10000
 NULL	[0]
 NULL	[1]
 NULL	[4]

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out?rev=781121&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out Tue Jun  2 18:51:20
2009
@@ -0,0 +1,103 @@
+query: DESCRIBE src_thrift
+aint	int	from deserializer
+astring	string	from deserializer
+lint	array<int>	from deserializer
+lstring	array<string>	from deserializer
+lintstring	array<org.apache.hadoop.hive.serde2.thrift.test.IntString>	from deserializer
+mstringstring	map<string,string>	from deserializer
+query: EXPLAIN
+SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF src_thrift s1) (TOK_TABREF src_thrift s2) (=
(. (TOK_TABLE_OR_COL s1) aint) (. (TOK_TABLE_OR_COL s2) aint)))) (TOK_INSERT (TOK_DESTINATION
(TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s1) aint)) (TOK_SELEXPR
(. (TOK_TABLE_OR_COL s2) lintstring)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        s2 
+            Select Operator
+              expressions:
+                    expr: aint
+                    type: int
+                    expr: lintstring
+                    type: array<struct<myint:int,mystring:string,underscore_int:int>>
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: int
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: int
+                tag: 1
+                value expressions:
+                      expr: 0
+                      type: int
+                      expr: 1
+                      type: array<struct<myint:int,mystring:string,underscore_int:int>>
+        s1 
+            Select Operator
+              expressions:
+                    expr: aint
+                    type: int
+              Reduce Output Operator
+                key expressions:
+                      expr: 0
+                      type: int
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 0
+                      type: int
+                tag: 0
+                value expressions:
+                      expr: 0
+                      type: int
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE.0}
+            1 {VALUE.0} {VALUE.1}
+          Select Operator
+            expressions:
+                  expr: 0
+                  type: int
+                  expr: 2
+                  type: array<struct<myint:int,mystring:string,underscore_int:int>>
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+query: SELECT s1.aint, s2.lintstring
+FROM src_thrift s1
+JOIN src_thrift s2
+ON s1.aint = s2.aint
+Input: default/src_thrift
+Output: /data/users/zshao/tools/495-trunk-apache-hive/ql/../build/ql/tmp/1611059524/151199080.10000
+-1952710710	[{"myint":25,"mystring":"125","underscore_int":5}]
+-1461153973	[{"myint":49,"mystring":"343","underscore_int":7}]
+-751827638	[{"myint":4,"mystring":"8","underscore_int":2}]
+-734328909	[{"myint":16,"mystring":"64","underscore_int":4}]
+0	null
+336964413	[{"myint":81,"mystring":"729","underscore_int":9}]
+465985200	[{"myint":1,"mystring":"1","underscore_int":1}]
+477111222	[{"myint":9,"mystring":"27","underscore_int":3}]
+1244525190	[{"myint":36,"mystring":"216","underscore_int":6}]
+1638581578	[{"myint":64,"mystring":"512","underscore_int":8}]
+1712634731	[{"myint":0,"mystring":"0","underscore_int":0}]

Modified: hadoop/hive/trunk/serde/if/test/complex.thrift
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/if/test/complex.thrift?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/if/test/complex.thrift (original)
+++ hadoop/hive/trunk/serde/if/test/complex.thrift Tue Jun  2 18:51:20 2009
@@ -3,6 +3,7 @@
 struct IntString {
   1: i32  myint;
   2: string myString;
+  3: i32  underscore_int;
 }
 
 struct Complex {

Modified: hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java
(original)
+++ hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde2/thrift/test/IntString.java
Tue Jun  2 18:51:20 2009
@@ -19,11 +19,13 @@
 public class IntString implements TBase, java.io.Serializable {
   public int myint;
   public String myString;
+  public int underscore_int;
 
   public final Isset __isset = new Isset();
   public static final class Isset implements java.io.Serializable {
     public boolean myint = false;
     public boolean myString = false;
+    public boolean underscore_int = false;
   }
 
   public IntString() {
@@ -31,13 +33,16 @@
 
   public IntString(
     int myint,
-    String myString)
+    String myString,
+    int underscore_int)
   {
     this();
     this.myint = myint;
     this.__isset.myint = true;
     this.myString = myString;
     this.__isset.myString = true;
+    this.underscore_int = underscore_int;
+    this.__isset.underscore_int = true;
   }
 
   public boolean equals(Object that) {
@@ -70,6 +75,15 @@
         return false;
     }
 
+    boolean this_present_underscore_int = true;
+    boolean that_present_underscore_int = true;
+    if (this_present_underscore_int || that_present_underscore_int) {
+      if (!(this_present_underscore_int && that_present_underscore_int))
+        return false;
+      if (this.underscore_int != that.underscore_int)
+        return false;
+    }
+
     return true;
   }
 
@@ -104,6 +118,14 @@
             TProtocolUtil.skip(iprot, field.type);
           }
           break;
+        case 3:
+          if (field.type == TType.I32) {
+            this.underscore_int = iprot.readI32();
+            this.__isset.underscore_int = true;
+          } else { 
+            TProtocolUtil.skip(iprot, field.type);
+          }
+          break;
         default:
           TProtocolUtil.skip(iprot, field.type);
           break;
@@ -131,6 +153,12 @@
       oprot.writeString(this.myString);
       oprot.writeFieldEnd();
     }
+    field.name = "underscore_int";
+    field.type = TType.I32;
+    field.id = 3;
+    oprot.writeFieldBegin(field);
+    oprot.writeI32(this.underscore_int);
+    oprot.writeFieldEnd();
     oprot.writeFieldStop();
     oprot.writeStructEnd();
   }
@@ -141,6 +169,8 @@
     sb.append(this.myint);
     sb.append(",myString:");
     sb.append(this.myString);
+    sb.append(",underscore_int:");
+    sb.append(this.underscore_int);
     sb.append(")");
     return sb.toString();
   }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
(original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
Tue Jun  2 18:51:20 2009
@@ -113,7 +113,7 @@
   }
   
   /**
-   * Get the standard ObjectInspector for an ObjectInspector.
+   * Get the corresponding standard ObjectInspector for an ObjectInspector.
    * 
    * The returned ObjectInspector can be used to inspect the standard object.
    */
@@ -144,14 +144,15 @@
       }
       case LIST: {
         ListObjectInspector loi = (ListObjectInspector)oi;
-        result = ObjectInspectorFactory.getStandardListObjectInspector(loi.getListElementObjectInspector());
+        result = ObjectInspectorFactory.getStandardListObjectInspector(
+            getStandardObjectInspector(loi.getListElementObjectInspector(), objectInspectorOption));
         break;
       }
       case MAP: {
         MapObjectInspector moi = (MapObjectInspector)oi;
         result = ObjectInspectorFactory.getStandardMapObjectInspector(
-            moi.getMapKeyObjectInspector(),
-            moi.getMapValueObjectInspector());
+            getStandardObjectInspector(moi.getMapKeyObjectInspector(), objectInspectorOption),
+            getStandardObjectInspector(moi.getMapValueObjectInspector(), objectInspectorOption));
         break;
       }
       case STRUCT: {
@@ -161,7 +162,7 @@
         List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fields.size());
         for(StructField f : fields) {
           fieldNames.add(f.getFieldName());
-          fieldObjectInspectors.add(f.getFieldObjectInspector());
+          fieldObjectInspectors.add(getStandardObjectInspector(f.getFieldObjectInspector(),
objectInspectorOption));
         }
         result = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
         break;

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
(original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
Tue Jun  2 18:51:20 2009
@@ -59,16 +59,20 @@
     private static class Token {
       public int position;
       public String text;
-      public boolean isAlphaDigit;
+      public boolean isType;
       public String toString() {
         return "" + position + ":" + text;
       }
     };
     
+    private static boolean isTypeChar(char c) {
+      return Character.isLetterOrDigit(c) || c == '_' || c == '.';
+    }
+    
     /**
      * Tokenize the typeInfoString.
-     * The rule is simple: all consecutive alphadigits are in one token, and all 
-     * other characters are one character per token.
+     * The rule is simple: all consecutive alphadigits and '_', '.' are in one
+     * token, and all other characters are one character per token.
      * 
      * tokenize("map<int,string>") should return ["map","<","int",",","string",">"]
      */
@@ -79,12 +83,12 @@
       while (end <= typeInfoString.length()) {
         // last character ends a token? 
         if (end == typeInfoString.length() 
-            || !Character.isLetterOrDigit(typeInfoString.charAt(end-1))
-            || !Character.isLetterOrDigit(typeInfoString.charAt(end))) {
+            || !isTypeChar(typeInfoString.charAt(end-1))
+            || !isTypeChar(typeInfoString.charAt(end))) {
           Token t = new Token();
           t.position = begin;
           t.text = typeInfoString.substring(begin, end);
-          t.isAlphaDigit = Character.isLetterOrDigit(typeInfoString.charAt(begin));
+          t.isType = isTypeChar(typeInfoString.charAt(begin));
           tokens.add(t);
           begin = end;
         }          
@@ -138,17 +142,17 @@
             && null == PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(t.text)
             && !t.text.equals(alternative)) {
           throw new IllegalArgumentException("Error: " + item + " expected at the position
"
-              + t.position + " of '" + typeInfoString + "'" );
+              + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found."
);
         }
       } else if (item.equals("name")) {
-        if (!t.isAlphaDigit && !t.text.equals(alternative)) {
+        if (!t.isType && !t.text.equals(alternative)) {
           throw new IllegalArgumentException("Error: " + item + " expected at the position
"
-              + t.position + " of '" + typeInfoString + "'" );
+              + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found."
);
         }
       } else {
         if (!item.equals(t.text) && !t.text.equals(alternative)) {
           throw new IllegalArgumentException("Error: " + item + " expected at the position
"
-              + t.position + " of '" + typeInfoString + "'" );
+              + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found."
);
         }
       }
       iToken ++;

Modified: hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
(original)
+++ hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
Tue Jun  2 18:51:20 2009
@@ -85,9 +85,10 @@
       assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
           PrimitiveObjectInspectorFactory.javaStringObjectInspector),
           fields.get(3).getFieldObjectInspector());
-      assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(
-          ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, 
-              ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)),
+      assertEquals(ObjectInspectorUtils.getStandardObjectInspector(
+              ObjectInspectorFactory.getStandardListObjectInspector(
+              ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, 
+              ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))),
           fields.get(4).getFieldObjectInspector());
       assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(
           PrimitiveObjectInspectorFactory.javaStringObjectInspector,

Modified: hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java?rev=781121&r1=781120&r2=781121&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java
(original)
+++ hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/thrift_test/CreateSequenceFile.java
Tue Jun  2 18:51:20 2009
@@ -106,7 +106,7 @@
       ArrayList<String> slist = new ArrayList<String>();
       slist.add("" + i*10); slist.add("" + i*100); slist.add("" + i*1000);
       ArrayList<IntString> islist = new ArrayList<IntString>();
-      islist.add(new IntString(i*i, ""+ i*i*i));
+      islist.add(new IntString(i*i, ""+ i*i*i, i));
       HashMap<String,String> hash = new HashMap<String,String>();
       hash.put("key_" + i, "value_" + i);
       



Mime
View raw message