hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nzh...@apache.org
Subject svn commit: r923585 - in /hadoop/hive/branches/branch-0.5: ./ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Tue, 16 Mar 2010 05:39:42 GMT
Author: nzhang
Date: Tue Mar 16 05:39:42 2010
New Revision: 923585

URL: http://svn.apache.org/viewvc?rev=923585&view=rev
Log:
HIVE-1085. ColumnarSerde should not be the default Serde when user specified a fileformat
using stored as (Yongqiang He via Ning Zhang)

Modified:
    hadoop/hive/branches/branch-0.5/CHANGES.txt
    hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/rcfile_default_format.q
    hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/rcfile_default_format.q.out

Modified: hadoop/hive/branches/branch-0.5/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/CHANGES.txt?rev=923585&r1=923584&r2=923585&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.5/CHANGES.txt Tue Mar 16 05:39:42 2010
@@ -30,6 +30,9 @@ Release 0.5.1 - Unreleased
     HIVE-1242. CombineHiveInputFormat does not work for compressed text files
     (Namit Jain via Ning Zhang)
 
+    HIVE-1085. ColumnarSerde should not be the default Serde when user
+    specified a fileformat using 'stored as' (Yongqiang He via Ning Zhang)
+
 Release 0.5.0
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=923585&r1=923584&r2=923585&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++ hadoop/hive/branches/branch-0.5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Tue Mar 16 05:39:42 2010
@@ -5374,8 +5374,8 @@ public class SemanticAnalyzer extends Ba
     String            mapKeyDelim   = null;
     String            lineDelim     = null;
     String            comment       = null;
-    String            inputFormat   = TEXTFILE_INPUT;
-    String            outputFormat  = TEXTFILE_OUTPUT;
+    String            inputFormat   = null;
+    String            outputFormat  = null;
     String            location      = null;
     String            serde         = null;
     Map<String, String> mapProp     = null;
@@ -5387,15 +5387,6 @@ public class SemanticAnalyzer extends Ba
     final int         CTAS          = 2;       // CREATE TABLE AS SELECT ... (CTAS)
     int               command_type  = CREATE_TABLE;
 
-    if ("SequenceFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT)))
{
-      inputFormat = SEQUENCEFILE_INPUT;
-      outputFormat = SEQUENCEFILE_OUTPUT;
-    } else if ("RCFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT)))
{
-      inputFormat = RCFILE_INPUT;
-      outputFormat = RCFILE_OUTPUT;
-      serde = COLUMNAR_SERDE;
-    }
-
     LOG.info("Creating table" + tableName + " positin=" + ast.getCharPositionInLine());
     int numCh = ast.getChildCount();
 
@@ -5531,6 +5522,21 @@ public class SemanticAnalyzer extends Ba
         default: assert false;
       }
     }
+    
+    if (inputFormat == null) {
+        assert outputFormat == null;
+        if ("SequenceFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT)))
{
+          inputFormat = SEQUENCEFILE_INPUT;
+          outputFormat = SEQUENCEFILE_OUTPUT;
+        } else if ("RCFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT)))
{
+          inputFormat = RCFILE_INPUT;
+          outputFormat = RCFILE_OUTPUT;
+          serde = COLUMNAR_SERDE;
+        } else {
+          inputFormat = TEXTFILE_INPUT;
+          outputFormat = TEXTFILE_OUTPUT;
+        }
+      }
 
     // check for existence of table
     if ( ifNotExists ) {

Modified: hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/rcfile_default_format.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/rcfile_default_format.q?rev=923585&r1=923584&r2=923585&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/rcfile_default_format.q
(original)
+++ hadoop/hive/branches/branch-0.5/ql/src/test/queries/clientpositive/rcfile_default_format.q
Tue Mar 16 05:39:42 2010
@@ -6,6 +6,15 @@ DESCRIBE EXTENDED rcfile_default_format;
 CREATE TABLE rcfile_default_format_ctas AS SELECT key,value FROM src;
 DESCRIBE EXTENDED rcfile_default_format_ctas; 
 
+CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE;
+INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src;
+DESCRIBE EXTENDED rcfile_default_format_txtfile; 
+
 SET hive.default.fileformat = TextFile;
 CREATE TABLE textfile_default_format_ctas AS SELECT key,value FROM rcfile_default_format_ctas;
-DESCRIBE EXTENDED textfile_default_format_ctas; 
\ No newline at end of file
+DESCRIBE EXTENDED textfile_default_format_ctas;
+
+DROP TABLE  rcfile_default_format;
+DROP TABLE  rcfile_default_format_ctas;
+DROP TABLE rcfile_default_format_txtfile;
+DROP TABLE textfile_default_format_ctas;

Modified: hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/rcfile_default_format.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/rcfile_default_format.q.out?rev=923585&r1=923584&r2=923585&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/rcfile_default_format.q.out
(original)
+++ hadoop/hive/branches/branch-0.5/ql/src/test/results/clientpositive/rcfile_default_format.q.out
Tue Mar 16 05:39:42 2010
@@ -9,7 +9,7 @@ POSTHOOK: query: DESCRIBE EXTENDED rcfil
 POSTHOOK: type: DESCTABLE
 key	string	from deserializer
 	 	 
-Detailed Table Information	Table(tableName:rcfile_default_format, dbName:default, owner:heyongqiang,
createTime:1257212499, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/rcfile_default_format,
inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1257212499})	
+Detailed Table Information	Table(tableName:rcfile_default_format, dbName:default, owner:heyongqiang,
createTime:1264196677, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/rcfile_default_format,
inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1264196677}, viewOriginalText:null, viewExpandedText:null)

 PREHOOK: query: CREATE TABLE rcfile_default_format_ctas AS SELECT key,value FROM src
 PREHOOK: type: CREATETABLE
 PREHOOK: Input: default@src
@@ -24,7 +24,27 @@ POSTHOOK: type: DESCTABLE
 key	string	from deserializer
 value	string	from deserializer
 	 	 
-Detailed Table Information	Table(tableName:rcfile_default_format_ctas, dbName:default, owner:heyongqiang,
createTime:1257212511, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/rcfile_default_format_ctas,
inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1257212511})	
+Detailed Table Information	Table(tableName:rcfile_default_format_ctas, dbName:default, owner:heyongqiang,
createTime:1264196683, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/rcfile_default_format_ctas,
inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1264196683}, viewOriginalText:null, viewExpandedText:null)

+PREHOOK: query: CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE rcfile_default_format_txtfile (key STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@rcfile_default_format_txtfile
+PREHOOK: query: INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@rcfile_default_format_txtfile
+POSTHOOK: query: INSERT OVERWRITE TABLE rcfile_default_format_txtfile SELECT key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@rcfile_default_format_txtfile
+PREHOOK: query: DESCRIBE EXTENDED rcfile_default_format_txtfile
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE EXTENDED rcfile_default_format_txtfile
+POSTHOOK: type: DESCTABLE
+key	string	
+	 	 
+Detailed Table Information	Table(tableName:rcfile_default_format_txtfile, dbName:default,
owner:heyongqiang, createTime:1264196683, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/rcfile_default_format_txtfile,
inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1264196683}, viewOriginalText:null, viewExpandedText:null)

 PREHOOK: query: CREATE TABLE textfile_default_format_ctas AS SELECT key,value FROM rcfile_default_format_ctas
 PREHOOK: type: CREATETABLE
 PREHOOK: Input: default@rcfile_default_format_ctas
@@ -39,4 +59,24 @@ POSTHOOK: type: DESCTABLE
 key	string	
 value	string	
 	 	 
-Detailed Table Information	Table(tableName:textfile_default_format_ctas, dbName:default,
owner:heyongqiang, createTime:1257212516, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/textfile_default_format_ctas,
inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1257212516})	
+Detailed Table Information	Table(tableName:textfile_default_format_ctas, dbName:default,
owner:heyongqiang, createTime:1264196692, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key,
type:string, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/heyongqiang/Documents/workspace/Hive-trunk/build/ql/test/data/warehouse/textfile_default_format_ctas,
inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[],
parameters:{transient_lastDdlTime=1264196692}, viewOriginalText:null, viewExpandedText:null)

+PREHOOK: query: DROP TABLE  rcfile_default_format
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE  rcfile_default_format
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_default_format
+PREHOOK: query: DROP TABLE  rcfile_default_format_ctas
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE  rcfile_default_format_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_default_format_ctas
+PREHOOK: query: DROP TABLE rcfile_default_format_txtfile
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE rcfile_default_format_txtfile
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@rcfile_default_format_txtfile
+PREHOOK: query: DROP TABLE textfile_default_format_ctas
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE textfile_default_format_ctas
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@textfile_default_format_ctas



Mime
View raw message