hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1234235 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ conf/ metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Sat, 21 Jan 2012 02:12:26 GMT
Author: hashutosh
Date: Sat Jan 21 02:12:26 2012
New Revision: 1234235

URL: http://svn.apache.org/viewvc?rev=1234235&view=rev
Log:
HIVE-2589 : Newly created partition should inherit properties from table (Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q
    hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q
    hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q
    hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
    hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
    hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml.template
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1234235&r1=1234234&r2=1234235&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Jan 21 02:12:26
2012
@@ -119,6 +119,7 @@ public class HiveConf extends Configurat
       HiveConf.ConfVars.METASTORE_EVENT_EXPIRY_DURATION,
       HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
       HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
+      HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
       };
 
   /**
@@ -296,6 +297,7 @@ public class HiveConf extends Configurat
     METASTORE_NON_TRANSACTIONAL_READ("javax.jdo.option.NonTransactionalRead", true),
     METASTORE_CONNECTION_USER_NAME("javax.jdo.option.ConnectionUserName", "APP"),
     METASTORE_END_FUNCTION_LISTENERS("hive.metastore.end.function.listeners", ""),
+    METASTORE_PART_INHERIT_TBL_PROPS("hive.metastore.partition.inherit.table.properties",""),
 
     // CLI
     CLIIGNOREERRORS("hive.cli.errors.ignore", false),

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1234235&r1=1234234&r2=1234235&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Sat Jan 21 02:12:26 2012
@@ -244,6 +244,12 @@
 </property>
 
 <property>
+  <name>hive.metastore.partition.inherit.table.properties</name>
+  <value></value>
+  <description>list of comma seperated keys occurring in table properties which will
get inherited to newly created partitions. * implies all the keys will get inherited.</description>
+</property>
+
+<property>
   <name>hive.metastore.end.function.listeners</name>
   <value></value>
   <description>list of comma separated listeners for the end of metastore functions.</description>

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1234235&r1=1234234&r2=1234235&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Sat
Jan 21 02:12:26 2012
@@ -26,14 +26,17 @@ import static org.apache.hadoop.hive.met
 import java.io.IOException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Formatter;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.Set;
 import java.util.Timer;
 import java.util.regex.Pattern;
 
@@ -1563,6 +1566,23 @@ public class HiveMetaStore extends Thrif
           part.putToParameters(Constants.DDL_TIME, Long.toString(time));
         }
 
+        // Inherit table properties into partition properties.
+        Map<String,String> tblParams = tbl.getParameters();
+        String inheritProps =  hiveConf.getVar(ConfVars.METASTORE_PART_INHERIT_TBL_PROPS).trim();
+        // Default value is empty string in which case no properties will be inherited.
+        // * implies all properties needs to be inherited
+        Set<String> inheritKeys = new HashSet<String>(Arrays.asList(inheritProps.split(",")));
+        if(inheritKeys.contains("*")){
+          inheritKeys =  tblParams.keySet();
+        }
+
+        for (String key : inheritKeys) {
+          String paramVal = tblParams.get(key);
+          if(null != paramVal){ // add the property only if it exists in table properties
+            part.putToParameters(key, paramVal);
+          }
+        }
+
         success = ms.addPartition(part);
 
       } finally {

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1234235&r1=1234234&r2=1234235&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Sat Jan 21 02:12:26 2012
@@ -360,7 +360,7 @@ public class QTestUtil {
         db.dropDatabase(dbName);
       }
     }
-    db.setCurrentDatabase(DEFAULT_DATABASE_NAME);
+    Hive.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
 
     List<String> roleNames = db.getAllRoleNames();
       for (String roleName : roleNames) {

Added: hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q?rev=1234235&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props.q Sat Jan 21 02:12:26
2012
@@ -0,0 +1,5 @@
+set hive.metastore.partition.inherit.table.properties=a,b;
+
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');

Added: hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q?rev=1234235&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_empty.q Sat Jan 21
02:12:26 2012
@@ -0,0 +1,4 @@
+set hive.metastore.partition.inherit.table.properties="";
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');

Added: hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q?rev=1234235&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/part_inherit_tbl_props_with_star.q Sat Jan
21 02:12:26 2012
@@ -0,0 +1,5 @@
+set hive.metastore.partition.inherit.table.properties=key1,*;
+
+create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties ('a'='myval','b'='yourval','c'='noval');
+alter table mytbl add partition (c2 = 'v1');
+describe formatted mytbl partition (c2='v1');

Added: hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out?rev=1234235&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props.q.out Sat Jan 21
02:12:26 2012
@@ -0,0 +1,47 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties
('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties
('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+c1                  	tinyint             	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+c2                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[v1]                	 
+Database:           	default             	 
+Table:              	mytbl               	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+	a                   	myval               
+	b                   	yourval             
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   

Added: hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out?rev=1234235&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_empty.q.out Sat Jan
21 02:12:26 2012
@@ -0,0 +1,45 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties
('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties
('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+c1                  	tinyint             	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+c2                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[v1]                	 
+Database:           	default             	 
+Table:              	mytbl               	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   

Added: hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out?rev=1234235&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/part_inherit_tbl_props_with_star.q.out Sat
Jan 21 02:12:26 2012
@@ -0,0 +1,48 @@
+PREHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties
('a'='myval','b'='yourval','c'='noval')
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table mytbl (c1 tinyint) partitioned by (c2 string) tblproperties
('a'='myval','b'='yourval','c'='noval')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mytbl
+PREHOOK: query: alter table mytbl add partition (c2 = 'v1')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@mytbl
+POSTHOOK: query: alter table mytbl add partition (c2 = 'v1')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@mytbl
+POSTHOOK: Output: default@mytbl@c2=v1
+PREHOOK: query: describe formatted mytbl partition (c2='v1')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted mytbl partition (c2='v1')
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+c1                  	tinyint             	None                
+	 	 
+# Partition Information	 	 
+# col_name            	data_type           	comment             
+	 	 
+c2                  	string              	None                
+	 	 
+# Detailed Partition Information	 	 
+Partition Value:    	[v1]                	 
+Database:           	default             	 
+Table:              	mytbl               	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+#### A masked pattern was here ####
+Partition Parameters:	 	 
+	a                   	myval               
+	b                   	yourval             
+	c                   	noval               
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
+InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+Storage Desc Params:	 	 
+	serialization.format	1                   



Mime
View raw message