hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r832059 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Mon, 02 Nov 2009 19:05:05 GMT
Author: namit
Date: Mon Nov  2 19:05:05 2009
New Revision: 832059

URL: http://svn.apache.org/viewvc?rev=832059&view=rev
Log:
HIVE-796. RCFile results missing columns from UNION ALL
(He Yongqiang via namit)


Added:
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_union.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_union.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=832059&r1=832058&r2=832059&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Nov  2 19:05:05 2009
@@ -228,6 +228,9 @@
     HIVE-902. Fix cli.sh to work with hadoop versions less than 20.
     (Carl Steinbach via zshao)
 
+    HIVE-796. RCFile results missing columns from UNION ALL
+    (He Yongqiang via namit)
+
 Release 0.4.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=832059&r1=832058&r2=832059&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Mon
Nov  2 19:05:05 2009
@@ -197,6 +197,35 @@
    * 
    */
   public static void setReadColumnIDs(Configuration conf, ArrayList<Integer> ids) {
+    String id = toReadColumnIDString(ids);
+    setReadColumnIDConf(conf, id);
+  }
+
+  /**
+   * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column
+   * is included in the list, RCFile's reader will not skip its value.
+   * 
+   */
+  public static void appendReadColumnIDs(Configuration conf, ArrayList<Integer> ids)
{
+    String id = toReadColumnIDString(ids);
+    String old = conf.get(READ_COLUMN_IDS_CONF_STR, null);
+    String newConfStr = id;
+    if(old !=null )
+      newConfStr = newConfStr + StringUtils.COMMA_STR + old;
+    
+    setReadColumnIDConf(conf, newConfStr);
+  }
+  
+  private static void setReadColumnIDConf(Configuration conf, String id) {
+    if (id == null || id.length() <= 0) {
+      conf.set(READ_COLUMN_IDS_CONF_STR, "");
+      return;
+    }
+
+    conf.set(READ_COLUMN_IDS_CONF_STR, id);
+  }
+
+  private static String toReadColumnIDString(ArrayList<Integer> ids) {
     String id = null;
     if (ids != null) {
       for (int i = 0; i < ids.size(); i++) {
@@ -207,13 +236,7 @@
         }
       }
     }
-
-    if (id == null || id.length() <= 0) {
-      conf.set(READ_COLUMN_IDS_CONF_STR, "");
-      return;
-    }
-
-    conf.set(READ_COLUMN_IDS_CONF_STR, id);
+    return id;
   }
 
   /**

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=832059&r1=832058&r2=832059&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Mon Nov
 2 19:05:05 2009
@@ -311,7 +311,7 @@
         TableScanOperator tableScan = (TableScanOperator) op;
         ArrayList<Integer> list = tableScan.getNeededColumnIDs();
         if (list != null)
-          HiveFileFormatUtils.setReadColumnIDs(jobConf, list);
+          HiveFileFormatUtils.appendReadColumnIDs(jobConf, list);
         else
           HiveFileFormatUtils.setFullyReadColumns(jobConf);
       }

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_union.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_union.q?rev=832059&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_union.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/rcfile_union.q Mon Nov  2 19:05:05
2009
@@ -0,0 +1,15 @@
+DROP TABLE rcfile_unionTable;
+CREATE table rcfile_unionTable (b STRING, c STRING)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+STORED AS RCFILE;
+
+FROM src
+INSERT OVERWRITE TABLE rcfile_unionTable SELECT src.key, src.value LIMIT 10;
+
+SELECT * FROM (
+SELECT b AS cola FROM rcfile_unionTable
+UNION ALL
+SELECT c AS cola FROM rcfile_unionTable) s;
+
+DROP TABLE rcfile_unionTable;
\ No newline at end of file

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_union.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_union.q.out?rev=832059&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_union.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/rcfile_union.q.out Mon Nov  2 19:05:05
2009
@@ -0,0 +1,64 @@
+PREHOOK: query: DROP TABLE rcfile_unionTable
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE rcfile_unionTable
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE table rcfile_unionTable (b STRING, c STRING)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE table rcfile_unionTable (b STRING, c STRING)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
+STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@rcfile_unionTable
+PREHOOK: query: FROM src
+INSERT OVERWRITE TABLE rcfile_unionTable SELECT src.key, src.value LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@rcfile_uniontable
+POSTHOOK: query: FROM src
+INSERT OVERWRITE TABLE rcfile_unionTable SELECT src.key, src.value LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@rcfile_uniontable
+PREHOOK: query: SELECT * FROM (
+SELECT b AS cola FROM rcfile_unionTable
+UNION ALL
+SELECT c AS cola FROM rcfile_unionTable) s
+PREHOOK: type: QUERY
+PREHOOK: Input: default@rcfile_uniontable
+PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1325887249/10000
+POSTHOOK: query: SELECT * FROM (
+SELECT b AS cola FROM rcfile_unionTable
+UNION ALL
+SELECT c AS cola FROM rcfile_unionTable) s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@rcfile_uniontable
+POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1325887249/10000
+val_238
+238
+val_86
+86
+val_311
+311
+val_27
+27
+val_165
+165
+val_409
+409
+val_255
+255
+val_278
+278
+val_98
+98
+val_484
+484
+PREHOOK: query: DROP TABLE rcfile_unionTable
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE rcfile_unionTable
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_uniontable



Mime
View raw message