Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 47FA9D5AB for ; Fri, 2 Nov 2012 11:21:06 +0000 (UTC) Received: (qmail 38775 invoked by uid 500); 2 Nov 2012 11:21:06 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 38549 invoked by uid 500); 2 Nov 2012 11:21:04 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 38227 invoked by uid 99); 2 Nov 2012 11:20:57 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 02 Nov 2012 11:20:57 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 02 Nov 2012 11:20:54 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 259CD23889C5 for ; Fri, 2 Nov 2012 11:20:34 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1404924 [3/6] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/ common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/had... Date: Fri, 02 Nov 2012 11:20:29 -0000 To: commits@hive.apache.org From: namit@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121102112034.259CD23889C5@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hive/trunk/ql/src/test/queries/clientnegative/column_rename5.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/column_rename5.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/column_rename5.q (original) +++ hive/trunk/ql/src/test/queries/clientnegative/column_rename5.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; CREATE TABLE skewedtable (key STRING, value STRING) SKEWED BY (key) ON (1,5,6); Modified: hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_col_name_value_no_mismatch.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_col_name_value_no_mismatch.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_col_name_value_no_mismatch.q (original) +++ hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_col_name_value_no_mismatch.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; CREATE TABLE skewed_table (key STRING, value STRING) SKEWED BY (key) ON ((1),(5,8),(6)); Modified: hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_dup_col_name.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_dup_col_name.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_dup_col_name.q (original) +++ hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_dup_col_name.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; CREATE TABLE skewed_table (key STRING, value STRING) SKEWED BY (key,key) ON ((1),(5),(6)); Modified: hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_failure_invalid_col_name.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_failure_invalid_col_name.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_failure_invalid_col_name.q (original) +++ hive/trunk/ql/src/test/queries/clientnegative/create_skewed_table_failure_invalid_col_name.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; CREATE TABLE skewed_table (key STRING, value STRING) SKEWED BY (key_non) ON ((1),(5),(6)); Added: hive/trunk/ql/src/test/queries/clientnegative/invalid_config1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/invalid_config1.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientnegative/invalid_config1.q (added) +++ hive/trunk/ql/src/test/queries/clientnegative/invalid_config1.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,4 @@ + +set hive.internal.ddl.list.bucketing.enable=true; + +CREATE TABLE skewedtable (key STRING, value STRING) SKEWED BY (key) ON (1,5,6); Added: hive/trunk/ql/src/test/queries/clientpositive/alter_skewed_table.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/alter_skewed_table.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/alter_skewed_table.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/alter_skewed_table.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,33 @@ +set hive.mapred.supports.subdirectories=true; +set hive.internal.ddl.list.bucketing.enable=true; + +create table original (key STRING, value STRING); + +describe formatted original; + +alter table original SKEWED BY (key) ON (1,5,6); + +describe formatted original; + +drop table original; + +create table original2 (key STRING, value STRING) ; + +describe formatted original2; + +alter table original2 SKEWED BY (key, value) ON ((1,1),(5,6)); + +describe formatted original2; + +drop table original2; + +create table original3 (key STRING, value STRING) SKEWED BY (key, value) ON ((1,1),(5,6)); + +describe formatted original3; + +alter table original3 not skewed; + +describe formatted original3; + +drop table original3; + Modified: hive/trunk/ql/src/test/queries/clientpositive/create_skewed_table1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/create_skewed_table1.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/create_skewed_table1.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/create_skewed_table1.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; CREATE TABLE list_bucket_single (key STRING, value STRING) SKEWED BY (key) ON ('1','5','6'); CREATE TABLE list_bucket_single_2 (key STRING, value STRING) SKEWED BY (key) ON ((1),(5),(6)); Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_1.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,82 @@ +set hive.mapred.supports.subdirectories=true; +set hive.internal.ddl.list.bucketing.enable=true; +set hive.optimize.listbucketing=true; +set mapred.input.dir.recursive=true; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) + +-- List bucketing query logic test case. We simulate the directory structure by DML here. +-- Test condition: +-- 1. where clause has multiple skewed columns +-- 2. where clause doesn't have non-skewed column +-- 3. where clause has one and operator +-- Test focus: +-- 1. basic list bucketing query work +-- Test result: +-- 1. pruner only pick up right directory +-- 2. query result is right + +-- create 1 table: fact_daily +-- 1. create a few partitions +-- 2. dfs move partition according to list bucketing structure (simulate DML) +-- $/fact_daily/ds=1/hr=4/x=../y=.. +-- notes: waste all partitions except ds=1 and hr=4 for list bucketing query test +-- 3. alter it to skewed table and set up location map +-- 4. list bucketing query +-- fact_daily (ds=1 and hr=4) will be used for list bucketing query +CREATE TABLE fact_daily(x int, y STRING) PARTITIONED BY (ds STRING, hr STRING) +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_daily'; + +-- create /fact_daily/ds=1/hr=1 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='1') +SELECT key, value FROM src WHERE key=484; + +-- create /fact_daily/ds=1/hr=2 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='2') +SELECT key, value FROM src WHERE key=369 or key=406; + +-- create /fact_daily/ds=1/hr=3 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='3') +SELECT key, value FROM src WHERE key=238; + +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=1 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=484/y=val_484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=2 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=3 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=238/y=val_238; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1; + +-- switch fact_daily to skewed table and point its location to /fact_daily/ds=1 +alter table fact_daily skewed by (x,y) on ((484,'val_484'),(238,'val_238')); +ALTER TABLE fact_daily ADD PARTITION (ds='1', hr='4'); + +-- set List Bucketing location map +alter table fact_daily PARTITION (ds = '1', hr='4') set skewed location ((484,'val_484')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=484/y=val_484', +(238,'val_238')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=238/y=val_238'); +describe formatted fact_daily PARTITION (ds = '1', hr='4'); + +SELECT * FROM fact_daily WHERE ds='1' and hr='4'; + +-- pruner only pick up skewed-value directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x FROM fact_daily WHERE ( ds='1' and hr='4') and (x=484 and y= 'val_484'); +-- List Bucketing Query +SELECT x FROM fact_daily WHERE ( ds='1' and hr='4') and (x=484 and y= 'val_484'); + +-- pruner only pick up skewed-value directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x,y FROM fact_daily WHERE ( ds='1' and hr='4') and (x=238 and y= 'val_238'); +-- List Bucketing Query +SELECT x,y FROM fact_daily WHERE ( ds='1' and hr='4') and (x=238 and y= 'val_238'); + +-- pruner only pick up default directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x FROM fact_daily WHERE ( ds='1' and hr='4') and (y = "3"); +-- List Bucketing Query +SELECT x FROM fact_daily WHERE ( ds='1' and hr='4') and (y = "3"); + +-- pruner only pick up default directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x,y FROM fact_daily WHERE ( ds='1' and hr='4') and x = 495; +-- List Bucketing Query +SELECT x,y FROM fact_daily WHERE ( ds='1' and hr='4') and x = 369; Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_2.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,79 @@ + set hive.mapred.supports.subdirectories=true; +set hive.internal.ddl.list.bucketing.enable=true; +set hive.optimize.listbucketing=true; +set mapred.input.dir.recursive=true; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) + +-- List bucketing query logic test case. We simulate the directory structure by DML here. +-- Test condition: +-- 1. where clause has multiple skewed columns and non-skewed columns +-- 3. where clause has a few operators +-- Test focus: +-- 1. basic list bucketing query work +-- Test result: +-- 1. pruner only pick up right directory +-- 2. query result is right + + +-- create 1 table: fact_daily +-- 1. create a few partitions +-- 2. dfs move partition according to list bucketing structure (simulate DML) +-- $/fact_daily/ds=1/hr=4/x=../y=.. +-- notes: waste all partitions except ds=1 and hr=4 for list bucketing query test +-- 3. alter it to skewed table and set up location map +-- 4. list bucketing query +-- fact_daily (ds=1 and hr=4) will be used for list bucketing query +CREATE TABLE fact_daily(x int, y STRING) PARTITIONED BY (ds STRING, hr STRING) +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_daily'; + +-- create /fact_daily/ds=1/hr=1 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='1') +SELECT key, value FROM src WHERE key=484; + +-- create /fact_daily/ds=1/hr=2 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='2') +SELECT key, value FROM src WHERE key=369 or key=406; + +-- create /fact_daily/ds=1/hr=3 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='3') +SELECT key, value FROM src WHERE key=238; + +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=1 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=484/y=val_484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=2 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=3 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=238/y=val_238; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1; + +-- switch fact_daily to skewed table and point its location to /fact_daily/ds=1 +alter table fact_daily skewed by (x,y) on ((484,'val_484'),(238,'val_238')); +ALTER TABLE fact_daily ADD PARTITION (ds='1', hr='4'); + +-- set List Bucketing location map +alter table fact_daily PARTITION (ds = '1', hr='4') set skewed location ((484,'val_484')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=484/y=val_484', +(238,'val_238')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4/x=238/y=val_238'); +describe formatted fact_daily PARTITION (ds = '1', hr='4'); + +SELECT * FROM fact_daily WHERE ds='1' and hr='4'; + +-- pruner only pick up default directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x,y FROM fact_daily WHERE ds='1' and hr='4' and y= 'val_484'; +-- List Bucketing Query +SELECT x,y FROM fact_daily WHERE ds='1' and hr='4' and y= 'val_484'; + +-- pruner only pick up default directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x FROM fact_daily WHERE ds='1' and hr='4' and x= 406; +-- List Bucketing Query +SELECT x,y FROM fact_daily WHERE ds='1' and hr='4' and x= 406; + +-- pruner only pick up skewed-value directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x,y FROM fact_daily WHERE ds='1' and hr='4' and ( (x=484 and y ='val_484') or (x=238 and y= 'val_238')) ; +-- List Bucketing Query +SELECT x,y FROM fact_daily WHERE ds='1' and hr='4' and ( (x=484 and y ='val_484') or (x=238 and y= 'val_238')) ; + +-- clean up +drop table fact_daily; \ No newline at end of file Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_multiskew_3.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,88 @@ +set hive.mapred.supports.subdirectories=true; +set hive.internal.ddl.list.bucketing.enable=true; +set hive.optimize.listbucketing=true; +set mapred.input.dir.recursive=true; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) + +-- List bucketing query logic test case. We simulate the directory structure by DML here. +-- Test condition: +-- 1. where clause has multiple skewed columns and non-skewed columns +-- 3. where clause has a few operators +-- Test focus: +-- 1. query works for on partition level. +-- A table can mix up non-skewed partition and skewed partition +-- Even for skewed partition, it can have different skewed information. +-- Test result: +-- 1. pruner only pick up right directory +-- 2. query result is right + +-- create 2 tables: fact_daily and fact_daily +-- fact_daily will be used for list bucketing query +-- fact_daily is a table used to prepare data and test directories +CREATE TABLE fact_daily(x int, y STRING, z STRING) PARTITIONED BY (ds STRING, hr STRING) +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_daily'; + +-- create /fact_daily/ds=1/hr=1 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='1') +SELECT key, value, value FROM src WHERE key=484; + +-- create /fact_daily/ds=1/hr=2 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='2') +SELECT key+11, value, value FROM src WHERE key=484; + +-- create /fact_daily/ds=1/hr=3 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='3') +SELECT key, value, value FROM src WHERE key=238; + +-- create /fact_daily/ds=1/hr=4 directory +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='1', hr='4') +SELECT key, value, value FROM src WHERE key=98; + +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=1 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/x=484/y=val_484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=2 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/x=495/y=val_484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=3 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/x=238/y=val_238; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=4 ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1; + +-- create a non-skewed partition ds=200 and hr =1 in fact_daily table +INSERT OVERWRITE TABLE fact_daily PARTITION (ds='200', hr='1') SELECT key, value, value FROM src WHERE key=145 or key=406 or key=429; + +-- switch fact_daily to skewed table, create partition ds=1 and hr=5 and point its location to /fact_daily/ds=1 +alter table fact_daily skewed by (x,y) on ((484,'val_484'),(238,'val_238')); +ALTER TABLE fact_daily SET TBLPROPERTIES('EXTERNAL'='TRUE'); +ALTER TABLE fact_daily ADD PARTITION (ds='1', hr='5') +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5'; + +-- set List Bucketing location map +alter table fact_daily PARTITION (ds = '1', hr='5') set skewed location ((484,'val_484')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/x=484/y=val_484', +(238,'val_238')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/x=238/y=val_238'); +describe formatted fact_daily PARTITION (ds = '1', hr='5'); + +-- alter skewed information and create partition ds=100 and hr=1 +alter table fact_daily skewed by (x,y) on ((495,'val_484')); +ALTER TABLE fact_daily ADD PARTITION (ds='100', hr='1') +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5'; +alter table fact_daily PARTITION (ds = '100', hr='1') set skewed location ((495,'val_484')='${hiveconf:hive.metastore.warehouse.dir}/fact_daily/ds=1/hr=5/x=495/y=val_484'); +describe formatted fact_daily PARTITION (ds = '100', hr='1'); + + +-- query non-skewed partition +explain extended +select * from fact_daily where ds='200' and hr='1' and x=145; +select * from fact_daily where ds='200' and hr='1' and x=145; +explain extended +select * from fact_daily where ds='200' and hr='1'; +select * from fact_daily where ds='200' and hr='1'; + +-- query skewed partition +explain extended +SELECT * FROM fact_daily WHERE ds='1' and hr='5' and (x=484 and y ='val_484'); +SELECT * FROM fact_daily WHERE ds='1' and hr='5' and (x=484 and y ='val_484'); + +-- query another skewed partition +explain extended +SELECT * FROM fact_daily WHERE ds='100' and hr='1' and (x=495 and y ='val_484'); +SELECT * FROM fact_daily WHERE ds='100' and hr='1' and (x=495 and y ='val_484'); Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_1.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,62 @@ +set hive.mapred.supports.subdirectories=true; +set hive.internal.ddl.list.bucketing.enable=true; +set hive.optimize.listbucketing=true; +set mapred.input.dir.recursive=true; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) + +-- List bucketing query logic test case. +-- Test condition: +-- 1. where clause has only one skewed column +-- 2. where clause doesn't have non-skewed column +-- 3. where clause has one and operator +-- Test result: +-- 1. pruner only pick up right directory +-- 2. query result is right + +-- create 2 tables: fact_daily and fact_tz +-- fact_daily will be used for list bucketing query +-- fact_tz is a table used to prepare data and test directories +CREATE TABLE fact_daily(x int) PARTITIONED BY (ds STRING); +CREATE TABLE fact_tz(x int) PARTITIONED BY (ds STRING, hr STRING) +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_tz'; + +-- create /fact_tz/ds=1/hr=1 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='1') +SELECT key FROM src WHERE key=484; + +-- create /fact_tz/ds=1/hr=2 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='2') +SELECT key+11 FROM src WHERE key=484; + +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=1 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=2 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1; + +-- switch fact_daily to skewed table and point its location to /fact_tz/ds=1 +alter table fact_daily skewed by (x) on (484); +ALTER TABLE fact_daily SET TBLPROPERTIES('EXTERNAL'='TRUE'); +ALTER TABLE fact_daily ADD PARTITION (ds='1') +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1'; + +-- set List Bucketing location map +alter table fact_daily PARTITION (ds = '1') set skewed location (484='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=484','HIVE_DEFAULT_LIST_BUCKETING_KEY'='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME'); +describe formatted fact_daily PARTITION (ds = '1'); + +SELECT * FROM fact_daily WHERE ds='1'; + +-- pruner only pick up skewed-value directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x FROM fact_daily WHERE ds='1' and x=484; +-- List Bucketing Query +SELECT x FROM fact_daily WHERE ds='1' and x=484; + +-- pruner only pick up default directory since x equal to non-skewed value +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x FROM fact_daily WHERE ds='1' and x=495; +-- List Bucketing Query +SELECT x FROM fact_daily WHERE ds='1' and x=495; +explain extended SELECT x FROM fact_daily WHERE ds='1' and x=1; +SELECT x FROM fact_daily WHERE ds='1' and x=1; Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_2.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,74 @@ +set hive.mapred.supports.subdirectories=true; +set hive.internal.ddl.list.bucketing.enable=true; +set hive.optimize.listbucketing=true; +set mapred.input.dir.recursive=true; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; + +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) + +-- List bucketing query logic test case. +-- Test condition: +-- 1. where clause has only one skewed column +-- 2. where clause doesn't have non-skewed column +-- Test focus: +-- 1. list bucketing query logic works fine for subquery +-- Test result: +-- 1. pruner only pick up right directory +-- 2. query result is right + +-- create 2 tables: fact_daily and fact_tz +-- fact_daily will be used for list bucketing query +-- fact_tz is a table used to prepare data and test directories +CREATE TABLE fact_daily(x int, y STRING) PARTITIONED BY (ds STRING); +CREATE TABLE fact_tz(x int, y STRING) PARTITIONED BY (ds STRING, hr STRING) +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_tz'; + +-- create /fact_tz/ds=1/hr=1 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='1') +SELECT key, value FROM src WHERE key=484; + +-- create /fact_tz/ds=1/hr=2 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='2') +SELECT key+11, value FROM src WHERE key=484; + +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=1 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=2 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1; + +-- switch fact_daily to skewed table and point its location to /fact_tz/ds=1 +alter table fact_daily skewed by (x) on (484); +ALTER TABLE fact_daily SET TBLPROPERTIES('EXTERNAL'='TRUE'); +ALTER TABLE fact_daily ADD PARTITION (ds='1') +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1'; + +-- set List Bucketing location map +alter table fact_daily PARTITION (ds = '1') set skewed location (484='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=484','HIVE_DEFAULT_LIST_BUCKETING_KEY'='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME'); +describe formatted fact_daily PARTITION (ds = '1'); + +SELECT * FROM fact_daily WHERE ds='1'; + +-- The first subquery +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended select x from (select x from fact_daily where ds = '1') subq where x = 484; +-- List Bucketing Query +select x from (select * from fact_daily where ds = '1') subq where x = 484; + +-- The second subquery +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended select x1, y1 from(select x as x1, y as y1 from fact_daily where ds ='1') subq where x1 = 484; +-- List Bucketing Query +select x1, y1 from(select x as x1, y as y1 from fact_daily where ds ='1') subq where x1 = 484; + + +-- The third subquery +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended select y, count(1) from fact_daily where ds ='1' and x = 484 group by y; +-- List Bucketing Query +select y, count(1) from fact_daily where ds ='1' and x = 484 group by y; + +-- The fourth subquery +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended select x, c from (select x, count(1) as c from fact_daily where ds = '1' group by x) subq where x = 484;; +-- List Bucketing Query +select x, c from (select x, count(1) as c from fact_daily where ds = '1' group by x) subq where x = 484; Added: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_query_oneskew_3.q Fri Nov 2 11:20:26 2012 @@ -0,0 +1,62 @@ +set hive.internal.ddl.list.bucketing.enable=true; +set hive.optimize.listbucketing=true; +set mapred.input.dir.recursive=true; +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.mapred.supports.subdirectories=true; + +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23) + +-- List bucketing query logic test case. +-- Test condition: +-- 1. where clause has single skewed columns and non-skewed columns +-- 3. where clause has a few operators +-- Test focus: +-- 1. basic list bucketing query works for not (equal) case +-- Test result: +-- 1. pruner only pick up right directory +-- 2. query result is right + +-- create 2 tables: fact_daily and fact_tz +-- fact_daily will be used for list bucketing query +-- fact_tz is a table used to prepare data and test directories +CREATE TABLE fact_daily(x int, y STRING, z STRING) PARTITIONED BY (ds STRING); +CREATE TABLE fact_tz(x int, y STRING, z STRING) PARTITIONED BY (ds STRING, hr STRING) +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_tz'; + +-- create /fact_tz/ds=1/hr=1 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='1') +SELECT key, value, value FROM src WHERE key=484; + +-- create /fact_tz/ds=1/hr=2 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='2') +SELECT key, value, value FROM src WHERE key=278 or key=86; + +-- create /fact_tz/ds=1/hr=3 directory +INSERT OVERWRITE TABLE fact_tz PARTITION (ds='1', hr='3') +SELECT key, value, value FROM src WHERE key=238; + +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=1 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=484; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=2 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME; +dfs -mv ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/hr=3 ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=238; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1; + +-- switch fact_daily to skewed table and point its location to /fact_tz/ds=1 +alter table fact_daily skewed by (x) on (484,238); +ALTER TABLE fact_daily SET TBLPROPERTIES('EXTERNAL'='TRUE'); +ALTER TABLE fact_daily ADD PARTITION (ds='1') +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1'; + +-- set List Bucketing location map +alter table fact_daily PARTITION (ds = '1') set skewed location (484='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=484', +238='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/x=238', +'HIVE_DEFAULT_LIST_BUCKETING_KEY'='${hiveconf:hive.metastore.warehouse.dir}/fact_tz/ds=1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME'); +describe formatted fact_daily PARTITION (ds = '1'); + +SELECT * FROM fact_daily WHERE ds='1'; + +-- pruner pick up right directory +-- explain plan shows which directory selected: Truncated Path -> Alias +explain extended SELECT x FROM fact_daily WHERE ds='1' and not (x = 86); +-- List Bucketing Query +SELECT x,y,z FROM fact_daily WHERE ds='1' and not (x = 86); Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt1.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt1.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt1.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt10.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt10.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt10.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt10.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt11.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt11.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt11.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt11.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt12.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt12.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt12.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt12.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt13.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt13.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt13.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt13.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt14.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt14.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt14.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt14.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt15.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt15.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt15.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt15.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt16.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt16.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt16.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt16.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt17.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt17.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt17.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt17.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt18.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt18.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt18.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt18.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt19.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt19.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt19.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt19.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt2.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt2.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt2.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt2.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt20.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt20.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt20.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt20.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt3.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt3.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt3.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt3.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt4.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt4.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt4.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt4.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt5.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt5.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt5.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt5.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt6.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt6.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt6.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt6.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt7.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt7.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt7.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt7.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt8.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt8.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt8.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt8.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Modified: hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt9.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt9.q?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt9.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/skewjoinopt9.q Fri Nov 2 11:20:26 2012 @@ -1,3 +1,4 @@ +set hive.mapred.supports.subdirectories=true; set hive.internal.ddl.list.bucketing.enable=true; set hive.optimize.skewjoin.compiletime = true; Added: hive/trunk/ql/src/test/results/clientnegative/column_change_skewedcol_type1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/column_change_skewedcol_type1.q.out?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/column_change_skewedcol_type1.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/column_change_skewedcol_type1.q.out Fri Nov 2 11:20:26 2012 @@ -0,0 +1,6 @@ +PREHOOK: query: CREATE TABLE skewedtable (key STRING, value STRING) SKEWED BY (key) ON (1,5,6) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE skewedtable (key STRING, value STRING) SKEWED BY (key) ON (1,5,6) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@skewedtable +FAILED: SemanticException key is a skewed column. It's not allowed to rename skewed column or change skewed column type. Modified: hive/trunk/ql/src/test/results/clientnegative/column_rename5.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/column_rename5.q.out?rev=1404924&r1=1404923&r2=1404924&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/column_rename5.q.out (original) +++ hive/trunk/ql/src/test/results/clientnegative/column_rename5.q.out Fri Nov 2 11:20:26 2012 @@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE skewedtable (key STRING, value STRING) SKEWED BY (key) ON (1,5,6) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@skewedtable -FAILED: SemanticException key is a skewed column. It's not allowed to rename skewed column. +FAILED: SemanticException key is a skewed column. It's not allowed to rename skewed column or change skewed column type. Added: hive/trunk/ql/src/test/results/clientnegative/invalid_config1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/invalid_config1.q.out?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientnegative/invalid_config1.q.out (added) +++ hive/trunk/ql/src/test/results/clientnegative/invalid_config1.q.out Fri Nov 2 11:20:26 2012 @@ -0,0 +1,2 @@ +FAILED: Hive Internal Error: hive.mapred.supports.subdirectories must be true if any one of following is true: hive.internal.ddl.list.bucketing.enable, hive.optimize.listbucketing and mapred.input.dir.recursive + Added: hive/trunk/ql/src/test/results/clientpositive/alter_skewed_table.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_skewed_table.q.out?rev=1404924&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/alter_skewed_table.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/alter_skewed_table.q.out Fri Nov 2 11:20:26 2012 @@ -0,0 +1,243 @@ +PREHOOK: query: create table original (key STRING, value STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table original (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@original +PREHOOK: query: describe formatted original +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted original +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table original SKEWED BY (key) ON (1,5,6) +PREHOOK: type: ALTERTABLE_SKEWED +PREHOOK: Input: default@original +PREHOOK: Output: default@original +POSTHOOK: query: alter table original SKEWED BY (key) ON (1,5,6) +POSTHOOK: type: ALTERTABLE_SKEWED +POSTHOOK: Input: default@original +POSTHOOK: Output: default@original +PREHOOK: query: describe formatted original +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted original +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Skewed Columns: [key] +Skewed Values: [[1], [5], [6]] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table original +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@original +PREHOOK: Output: default@original +POSTHOOK: query: drop table original +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@original +POSTHOOK: Output: default@original +PREHOOK: query: create table original2 (key STRING, value STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table original2 (key STRING, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@original2 +PREHOOK: query: describe formatted original2 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted original2 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table original2 SKEWED BY (key, value) ON ((1,1),(5,6)) +PREHOOK: type: ALTERTABLE_SKEWED +PREHOOK: Input: default@original2 +PREHOOK: Output: default@original2 +POSTHOOK: query: alter table original2 SKEWED BY (key, value) ON ((1,1),(5,6)) +POSTHOOK: type: ALTERTABLE_SKEWED +POSTHOOK: Input: default@original2 +POSTHOOK: Output: default@original2 +PREHOOK: query: describe formatted original2 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted original2 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Skewed Columns: [key, value] +Skewed Values: [[1, 1], [5, 6]] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table original2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@original2 +PREHOOK: Output: default@original2 +POSTHOOK: query: drop table original2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@original2 +POSTHOOK: Output: default@original2 +PREHOOK: query: create table original3 (key STRING, value STRING) SKEWED BY (key, value) ON ((1,1),(5,6)) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table original3 (key STRING, value STRING) SKEWED BY (key, value) ON ((1,1),(5,6)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@original3 +PREHOOK: query: describe formatted original3 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted original3 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Skewed Columns: [key, value] +Skewed Values: [[1, 1], [5, 6]] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: alter table original3 not skewed +PREHOOK: type: ALTERTABLE_SKEWED +PREHOOK: Input: default@original3 +PREHOOK: Output: default@original3 +POSTHOOK: query: alter table original3 not skewed +POSTHOOK: type: ALTERTABLE_SKEWED +POSTHOOK: Input: default@original3 +POSTHOOK: Output: default@original3 +PREHOOK: query: describe formatted original3 +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted original3 +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table original3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@original3 +PREHOOK: Output: default@original3 +POSTHOOK: query: drop table original3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@original3 +POSTHOOK: Output: default@original3