hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject hive git commit: HIVE-13064 : Fix passing serde properties from insert overwrite directory (Rajat Khandelwal, reviewed by Amareshwari Sriramadasu)
Date Wed, 24 Feb 2016 10:33:40 GMT
Repository: hive
Updated Branches:
  refs/heads/master 8c0d148cd -> 402fabe0c


HIVE-13064 : Fix passing serde properties from insert overwrite directory (Rajat Khandelwal,
reviewed by Amareshwari Sriramadasu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/402fabe0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/402fabe0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/402fabe0

Branch: refs/heads/master
Commit: 402fabe0c45a9c0e3b1a2d63d68fca8f8e399fad
Parents: 8c0d148
Author: Rajat Khandelwal <prongs@apache.org>
Authored: Wed Feb 24 16:03:14 2016 +0530
Committer: Amareshwari Sriramadasu <amareshwari@apache.org>
Committed: Wed Feb 24 16:03:14 2016 +0530

----------------------------------------------------------------------
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  6 ++++-
 .../apache/hadoop/hive/ql/plan/PlanUtils.java   |  3 +++
 .../clientpositive/insert_overwrite_directory.q |  8 +++----
 .../insert_overwrite_directory.q.out            | 24 ++++++++++----------
 4 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/402fabe0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 8a06582..7d2595d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2094,7 +2094,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
                   ASTNode serdeChild = (ASTNode) child.getChild(0);
                   storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText()));
                   directoryDesc.setSerName(storageFormat.getSerde());
-                  directoryDescIsSet=true;
+                  if (serdeChild.getChildCount() > 1) {
+                    directoryDesc.setSerdeProps(new HashMap<String, String>());
+                    readProps((ASTNode) serdeChild.getChild(1).getChild(0), directoryDesc.getSerdeProps());
+                  }
+                  directoryDescIsSet = true;
                   break;
               }
             }

http://git-wip-us.apache.org/repos/asf/hive/blob/402fabe0/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 04d26f3..990a10c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -141,6 +141,9 @@ public final class PlanUtils {
         properties.setProperty(
             serdeConstants.SERIALIZATION_LIB, directoryDesc.getSerName());
       }
+      if (directoryDesc.getSerdeProps() != null) {
+        properties.putAll(directoryDesc.getSerdeProps());
+      }
       if (directoryDesc.getOutputFormat() != null){
         ret.setOutputFileFormatClass(JavaUtils.loadClass(directoryDesc.getOutputFormat()));
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/402fabe0/ql/src/test/queries/clientpositive/insert_overwrite_directory.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/insert_overwrite_directory.q b/ql/src/test/queries/clientpositive/insert_overwrite_directory.q
index c3d419b..2775314 100644
--- a/ql/src/test/queries/clientpositive/insert_overwrite_directory.q
+++ b/ql/src/test/queries/clientpositive/insert_overwrite_directory.q
@@ -80,8 +80,8 @@ insert overwrite directory '../../data/files/array_table_4'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
 WITH SERDEPROPERTIES (
 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol',
-'quote.delim'= '("|\\[|\\])',  'field.delim'=', ',
-'serialization.null.format'='-'  ) STORED AS TEXTFILE 
+'quote.delim'= '("|\\[|\\])',  'field.delim'=',',
+'serialization.null.format'='-NA-', 'colelction.delim'='#') STORED AS TEXTFILE
 select a, null, b from array_table;
 
 dfs -cat ../../data/files/array_table_4/000000_0;
@@ -97,8 +97,8 @@ insert overwrite directory '../../data/files/map_table_4'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
 WITH SERDEPROPERTIES (
 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol',
-'quote.delim'= '("|\\[|\\])',  'field.delim'=', ',
-'serialization.null.format'='-'  ) STORED AS TEXTFILE 
+'quote.delim'= '("|\\[|\\])',  'field.delim'=':',
+'serialization.null.format'='-NA-', 'colelction.delim'='#', 'mapkey.delim'='%') STORED AS
TEXTFILE
 select foo, null, bar from map_table;
 
 dfs -cat ../../data/files/map_table_4/000000_0;

http://git-wip-us.apache.org/repos/asf/hive/blob/402fabe0/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out b/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out
index e5d2455..a459839 100644
--- a/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out
+++ b/ql/src/test/results/clientpositive/insert_overwrite_directory.q.out
@@ -1190,8 +1190,8 @@ PREHOOK: query: insert overwrite directory '../../data/files/array_table_4'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
 WITH SERDEPROPERTIES (
 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol',
-'quote.delim'= '("|\\[|\\])',  'field.delim'=', ',
-'serialization.null.format'='-'  ) STORED AS TEXTFILE 
+'quote.delim'= '("|\\[|\\])',  'field.delim'=',',
+'serialization.null.format'='-NA-', 'colelction.delim'='#') STORED AS TEXTFILE
 select a, null, b from array_table
 PREHOOK: type: QUERY
 PREHOOK: Input: default@array_table
@@ -1200,14 +1200,14 @@ POSTHOOK: query: insert overwrite directory '../../data/files/array_table_4'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
 WITH SERDEPROPERTIES (
 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol',
-'quote.delim'= '("|\\[|\\])',  'field.delim'=', ',
-'serialization.null.format'='-'  ) STORED AS TEXTFILE 
+'quote.delim'= '("|\\[|\\])',  'field.delim'=',',
+'serialization.null.format'='-NA-', 'colelction.delim'='#') STORED AS TEXTFILE
 select a, null, b from array_table
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@array_table
 #### A masked pattern was here ####
-a1a2a3\Nb1b2b3b4
-a21a22a23\Nb21b22b23b24
+a1#a2#a3,-NA-,b1#b2#b3#b4
+a21#a22#a23,-NA-,b21#b22#b23#b24
 PREHOOK: query: insert overwrite directory '../../data/files/map_table_3'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.DelimitedJSONSerDe'
 STORED AS TEXTFILE
@@ -1228,8 +1228,8 @@ PREHOOK: query: insert overwrite directory '../../data/files/map_table_4'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
 WITH SERDEPROPERTIES (
 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol',
-'quote.delim'= '("|\\[|\\])',  'field.delim'=', ',
-'serialization.null.format'='-'  ) STORED AS TEXTFILE 
+'quote.delim'= '("|\\[|\\])',  'field.delim'=':',
+'serialization.null.format'='-NA-', 'colelction.delim'='#', 'mapkey.delim'='%') STORED AS
TEXTFILE
 select foo, null, bar from map_table
 PREHOOK: type: QUERY
 PREHOOK: Input: default@map_table
@@ -1238,14 +1238,14 @@ POSTHOOK: query: insert overwrite directory '../../data/files/map_table_4'
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
 WITH SERDEPROPERTIES (
 'serialization.format'= 'org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol',
-'quote.delim'= '("|\\[|\\])',  'field.delim'=', ',
-'serialization.null.format'='-'  ) STORED AS TEXTFILE 
+'quote.delim'= '("|\\[|\\])',  'field.delim'=':',
+'serialization.null.format'='-NA-', 'colelction.delim'='#', 'mapkey.delim'='%') STORED AS
TEXTFILE
 select foo, null, bar from map_table
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@map_table
 #### A masked pattern was here ####
-foo1\Nk1v1k2v2k3v3
-foo2\Nk21v21k22v22k31v31
+foo1:-NA-:k1%v1#k2%v2#k3%v3
+foo2:-NA-:k21%v21#k22%v22#k31%v31
 PREHOOK: query: insert overwrite directory '../../data/files/rctable'
 STORED AS RCFILE
 select value,key from src


Mime
View raw message