hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r736092 - in /hadoop/hive/trunk: CHANGES.txt data/files/null.txt ql/src/test/queries/clientpositive/input21.q ql/src/test/results/clientpositive/input21.q.out serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java
Date Tue, 20 Jan 2009 20:27:15 GMT
Author: zshao
Date: Tue Jan 20 12:27:15 2009
New Revision: 736092

URL: http://svn.apache.org/viewvc?rev=736092&view=rev
Log:
HIVE-235. Fixed DynamicSerDe to work with null values with Thrift Protocols that can have
missing fields for null values. (zshao)

Added:
    hadoop/hive/trunk/data/files/null.txt
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/input21.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/input21.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=736092&r1=736091&r2=736092&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Jan 20 12:27:15 2009
@@ -80,8 +80,11 @@
 
   BUG FIXES
 
-    HIVE-222. Fixed Group by on a combination of disitinct and non distinct aggregates.
-    (Ashish Thusoo via zshao)
+    HIVE-235. Fixed DynamicSerDe to work with null values with Thrift
+    Protocols that can have missing fields for null values. (zshao)
+
+    HIVE-222. Fixed Group by on a combination of disitinct and non distinct
+    aggregates. (Ashish Thusoo via zshao)
 
     HIVE-161. Fixed xpath x.y when x is a null list. (zshao)
 

Added: hadoop/hive/trunk/data/files/null.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/null.txt?rev=736092&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/null.txt (added)
+++ hadoop/hive/trunk/data/files/null.txt Tue Jan 20 12:27:15 2009
@@ -0,0 +1,10 @@
+1.01same0
+1.01same1
+1.01same2
+1.01same3
+1.01same4
+\N1same5
+\N\Nsame6
+1.0\Nsame7
+1.01same8
+1.01same9

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/input21.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/input21.q?rev=736092&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/input21.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/input21.q Tue Jan 20 12:27:15 2009
@@ -0,0 +1,10 @@
+DROP TABLE src_null;
+
+CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../data/files/null.txt' INTO TABLE src_null;
+
+EXPLAIN SELECT * FROM src_null DISTRIBUTE BY c SORT BY d;
+
+SELECT * FROM src_null DISTRIBUTE BY c SORT BY d;
+
+DROP TABLE src_null;

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/input21.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/input21.q.out?rev=736092&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/input21.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/input21.q.out Tue Jan 20 12:27:15
2009
@@ -0,0 +1,63 @@
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_TABREF src_null)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_DISTRIBUTEBY c) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC
d))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        src_null 
+            Select Operator
+              expressions:
+                    expr: a
+                    type: string
+                    expr: b
+                    type: string
+                    expr: c
+                    type: string
+                    expr: d
+                    type: string
+              Reduce Output Operator
+                key expressions:
+                      expr: 3
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: 2
+                      type: string
+                tag: -1
+                value expressions:
+                      expr: 0
+                      type: string
+                      expr: 1
+                      type: string
+                      expr: 2
+                      type: string
+                      expr: 3
+                      type: string
+      Reduce Operator Tree:
+        Extract
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.TextInputFormat
+                output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+
+1.0	1	same	0
+1.0	1	same	1
+1.0	1	same	2
+1.0	1	same	3
+1.0	1	same	4
+NULL	1	same	5
+NULL	NULL	same	6
+1.0	NULL	same	7
+1.0	1	same	8
+1.0	1	same	9

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java?rev=736092&r1=736091&r2=736092&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java
(original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java
Tue Jan 20 12:27:15 2009
@@ -119,17 +119,18 @@
    */
   protected boolean isRealThrift = false;
 
+  protected boolean[] fieldsPresent; 
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException,
IllegalAccessException {
     ArrayList<Object> struct = null;
 
     if (reuse == null) {
       struct = new ArrayList<Object>(this.getNumFields());
-      for(int i=0; i<this.getNumFields(); i++) {
+      for(int i=0; i<ordered_types.length; i++) {
         struct.add(null);
       }
     } else {
       struct = (ArrayList<Object>) reuse;
-      assert(struct.size() == this.getNumFields());
+      assert(struct.size() == ordered_types.length);
     }
 
     boolean fastSkips = iprot instanceof org.apache.hadoop.hive.serde2.thrift.SkippableTProtocol;
@@ -137,6 +138,11 @@
     // may need to strip away the STOP marker when in thrift mode
     boolean stopSeen = false;
 
+    if (fieldsPresent == null) {
+      fieldsPresent = new boolean[ordered_types.length];
+    }
+    Arrays.fill(fieldsPresent, false);
+
     // Read the fields.
     for(int i = 0; i < this.getNumFields(); i++) {
       DynamicSerDeTypeBase mt = null;
@@ -189,7 +195,15 @@
       if(thrift_mode) {
         iprot.readFieldEnd();
       }
+      fieldsPresent[orderedId] = true;
     }
+    
+    for(int i = 0; i < ordered_types.length; i++) {
+      if (!fieldsPresent[i]) {
+        struct.set(i, null);
+      }
+    }
+    
     if(thrift_mode && !stopSeen) {
       // strip off the STOP marker, which may be left if all the fields were in the serialization
       iprot.readFieldBegin();



Mime
View raw message