hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1610880 - in /hive/trunk: itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/ ql/src/test/results/clientpositive/tez/
Date Tue, 15 Jul 2014 23:09:31 GMT
Author: hashutosh
Date: Tue Jul 15 23:09:31 2014
New Revision: 1610880

URL: http://svn.apache.org/r1610880
Log:
HIVE-7262 : Partitioned Table Function (PTF) query fails on ORC table when attempting to vectorize
(Matt McCline via Jitendra Nath Pandey)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/vectorized_ptf.q
    hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out
Modified:
    hive/trunk/itests/qtest/testconfiguration.properties
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

Modified: hive/trunk/itests/qtest/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/testconfiguration.properties?rev=1610880&r1=1610879&r2=1610880&view=diff
==============================================================================
--- hive/trunk/itests/qtest/testconfiguration.properties (original)
+++ hive/trunk/itests/qtest/testconfiguration.properties Tue Jul 15 23:09:31 2014
@@ -1,5 +1,5 @@
 minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q
 minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q
 minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q
-minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t
 able.q
+minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t
 able.q,vectorized_ptf.q
 beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q,
 exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwr
 ite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1610880&r1=1610879&r2=1610880&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Tue
Jul 15 23:09:31 2014
@@ -641,6 +641,16 @@ public class Vectorizer implements Physi
   }
 
   private boolean validateExprNodeDescRecursive(ExprNodeDesc desc) {
+    if (desc instanceof ExprNodeColumnDesc) {
+      ExprNodeColumnDesc c = (ExprNodeColumnDesc) desc;
+      // Currently, we do not support vectorized virtual columns (see HIVE-5570).
+      for (VirtualColumn vc : VirtualColumn.VIRTUAL_COLUMNS) {
+        if (c.getColumn().equals(vc.getName())) {
+            LOG.info("Cannot vectorize virtual column " + c.getColumn());
+            return false;
+        }
+      }
+    }
     String typeName = desc.getTypeInfo().getTypeName();
     boolean ret = validateDataType(typeName);
     if (!ret) {
@@ -724,6 +734,7 @@ public class Vectorizer implements Physi
     Map<String, Integer> cmap = new HashMap<String, Integer>();
     int columnCount = 0;
     for (ColumnInfo c : rs.getSignature()) {
+      // Earlier, validation code should have eliminated virtual columns usage (HIVE-5560).
       if (!isVirtualColumn(c)) {
         cmap.put(c.getInternalName(), columnCount++);
       }

Added: hive/trunk/ql/src/test/queries/clientpositive/vectorized_ptf.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vectorized_ptf.q?rev=1610880&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vectorized_ptf.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/vectorized_ptf.q Tue Jul 15 23:09:31 2014
@@ -0,0 +1,609 @@
+SET hive.vectorized.execution.enabled=true;
+
+DROP TABLE part_staging;
+DROP TABLE part;
+
+-- NOTE: This test is a copy of ptf.
+-- NOTE: We cannot vectorize "pure" table functions (e.g. NOOP) -- given their blackbox nature.
So only queries without table functions and
+-- NOTE: with windowing will be vectorized.
+
+-- data setup
+CREATE TABLE part_staging( 
+    p_partkey INT,
+    p_name STRING,
+    p_mfgr STRING,
+    p_brand STRING,
+    p_type STRING,
+    p_size INT,
+    p_container STRING,
+    p_retailprice DOUBLE,
+    p_comment STRING
+);
+
+LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part_staging;
+
+CREATE TABLE part( 
+    p_partkey INT,
+    p_name STRING,
+    p_mfgr STRING,
+    p_brand STRING,
+    p_type STRING,
+    p_size INT,
+    p_container STRING,
+    p_retailprice DOUBLE,
+    p_comment STRING
+) STORED AS ORC;
+
+DESCRIBE EXTENDED part;
+
+insert into table part select * from part_staging;
+
+--1. test1
+
+explain extended
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noop(on part 
+  partition by p_mfgr
+  order by p_name
+  );
+
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noop(on part 
+  partition by p_mfgr
+  order by p_name
+  );
+
+-- 2. testJoinWithNoop
+
+explain extended
+select p_mfgr, p_name,
+p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz
+from noop (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j
+distribute by j.p_mfgr
+sort by j.p_name)
+;
+
+select p_mfgr, p_name,
+p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz
+from noop (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j
+distribute by j.p_mfgr
+sort by j.p_name)
+;    
+
+-- 3. testOnlyPTF
+
+explain extended
+select p_mfgr, p_name, p_size
+from noop(on part
+partition by p_mfgr
+order by p_name);
+
+select p_mfgr, p_name, p_size
+from noop(on part
+partition by p_mfgr
+order by p_name);
+
+-- 4. testPTFAlias
+
+explain extended
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noop(on part 
+  partition by p_mfgr
+  order by p_name
+  ) abc;
+
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noop(on part 
+  partition by p_mfgr
+  order by p_name
+  ) abc;
+
+-- 5. testPTFAndWhereWithWindowing
+
+explain extended
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz

+from noop(on part 
+          partition by p_mfgr 
+          order by p_name 
+          ) 
+;
+
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz

+from noop(on part 
+          partition by p_mfgr 
+          order by p_name 
+          ) 
+;
+
+-- 6. testSWQAndPTFAndGBy
+
+explain extended
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz

+from noop(on part 
+          partition by p_mfgr 
+          order by p_name 
+          ) 
+group by p_mfgr, p_name, p_size  
+;
+
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz

+from noop(on part 
+          partition by p_mfgr 
+          order by p_name 
+          ) 
+group by p_mfgr, p_name, p_size  
+;
+
+-- 7. testJoin
+
+explain extended
+select abc.* 
+from noop(on part 
+partition by p_mfgr 
+order by p_name 
+) abc join part p1 on abc.p_partkey = p1.p_partkey;
+
+select abc.* 
+from noop(on part 
+partition by p_mfgr 
+order by p_name 
+) abc join part p1 on abc.p_partkey = p1.p_partkey;
+
+-- 8. testJoinRight
+
+explain extended
+select abc.* 
+from part p1 join noop(on part 
+partition by p_mfgr 
+order by p_name 
+) abc on abc.p_partkey = p1.p_partkey;
+
+select abc.* 
+from part p1 join noop(on part 
+partition by p_mfgr 
+order by p_name 
+) abc on abc.p_partkey = p1.p_partkey;
+
+-- 9. testNoopWithMap
+
+explain extended
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name, p_size desc) as r
+from noopwithmap(on part
+partition by p_mfgr
+order by p_name, p_size desc);
+
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name, p_size desc) as r
+from noopwithmap(on part
+partition by p_mfgr
+order by p_name, p_size desc);
+
+-- 10. testNoopWithMapWithWindowing 
+
+explain extended
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noopwithmap(on part 
+  partition by p_mfgr
+  order by p_name);
+  
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noopwithmap(on part 
+  partition by p_mfgr
+  order by p_name);
+  
+-- 11. testHavingWithWindowingPTFNoGBY
+
+explain extended
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noop(on part
+partition by p_mfgr
+order by p_name)
+;
+
+select p_mfgr, p_name, p_size,
+rank() over (partition by p_mfgr order by p_name) as r,
+dense_rank() over (partition by p_mfgr order by p_name) as dr,
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row) as s1
+from noop(on part
+partition by p_mfgr
+order by p_name)
+;
+  
+-- 12. testFunctionChain
+
+explain extended
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row)  as s1
+from noop(on noopwithmap(on noop(on part 
+partition by p_mfgr 
+order by p_mfgr, p_name
+)));
+
+select p_mfgr, p_name, p_size, 
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row)  as s1
+from noop(on noopwithmap(on noop(on part 
+partition by p_mfgr 
+order by p_mfgr, p_name
+)));
+ 
+-- 13. testPTFAndWindowingInSubQ
+
+explain extended
+select p_mfgr, p_name, 
+sub1.cd, sub1.s1 
+from (select p_mfgr, p_name, 
+count(p_size) over (partition by p_mfgr order by p_name) as cd, 
+p_retailprice, 
+sum(p_retailprice) over w1  as s1
+from noop(on part 
+partition by p_mfgr 
+order by p_name) 
+window w1 as (partition by p_mfgr order by p_name rows between 2 preceding and 2 following)

+) sub1 ;
+
+select p_mfgr, p_name, 
+sub1.cd, sub1.s1 
+from (select p_mfgr, p_name, 
+count(p_size) over (partition by p_mfgr order by p_name) as cd, 
+p_retailprice, 
+sum(p_retailprice) over w1  as s1
+from noop(on part 
+partition by p_mfgr 
+order by p_name) 
+window w1 as (partition by p_mfgr order by p_name rows between 2 preceding and 2 following)

+) sub1 ;
+
+-- 14. testPTFJoinWithWindowingWithCount
+
+explain extended
+select abc.p_mfgr, abc.p_name, 
+rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, 
+dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, 
+count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, 
+abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name
rows between unbounded preceding and current row) as s1, 
+abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort
by abc.p_name) as deltaSz 
+from noop(on part 
+partition by p_mfgr 
+order by p_name 
+) abc join part p1 on abc.p_partkey = p1.p_partkey 
+;
+
+select abc.p_mfgr, abc.p_name, 
+rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, 
+dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, 
+count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, 
+abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name
rows between unbounded preceding and current row) as s1, 
+abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort
by abc.p_name) as deltaSz 
+from noop(on part 
+partition by p_mfgr 
+order by p_name 
+) abc join part p1 on abc.p_partkey = p1.p_partkey 
+;
+
+-- 15. testDistinctInSelectWithPTF
+
+explain extended
+select DISTINCT p_mfgr, p_name, p_size 
+from noop(on part 
+partition by p_mfgr 
+order by p_name);
+
+select DISTINCT p_mfgr, p_name, p_size 
+from noop(on part 
+partition by p_mfgr 
+order by p_name);
+
+  
+-- 16. testViewAsTableInputToPTF
+create view IF NOT EXISTS mfgr_price_view as 
+select p_mfgr, p_brand, 
+sum(p_retailprice) as s 
+from part 
+group by p_mfgr, p_brand;
+
+explain extended
+select p_mfgr, p_brand, s, 
+sum(s) over w1  as s1
+from noop(on mfgr_price_view 
+partition by p_mfgr 
+order by p_mfgr)  
+window w1 as ( partition by p_mfgr order by p_brand rows between 2 preceding and current
row);
+
+select p_mfgr, p_brand, s, 
+sum(s) over w1  as s1
+from noop(on mfgr_price_view 
+partition by p_mfgr 
+order by p_mfgr)  
+window w1 as ( partition by p_mfgr order by p_brand rows between 2 preceding and current
row);
+  
+-- 17. testMultipleInserts2SWQsWithPTF
+CREATE TABLE part_4( 
+p_mfgr STRING, 
+p_name STRING, 
+p_size INT, 
+r INT, 
+dr INT, 
+s DOUBLE);
+
+CREATE TABLE part_5( 
+p_mfgr STRING, 
+p_name STRING, 
+p_size INT, 
+s2 INT, 
+r INT, 
+dr INT, 
+cud DOUBLE, 
+fv1 INT);
+
+explain extended
+from noop(on part 
+partition by p_mfgr 
+order by p_name) 
+INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, 
+rank() over (distribute by p_mfgr sort by p_name) as r, 
+dense_rank() over (distribute by p_mfgr sort by p_name) as dr, 
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding
and current row)  as s  
+INSERT OVERWRITE TABLE part_5 select  p_mfgr,p_name, p_size,  
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and
current row),1) as s2,
+rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as r, 
+dense_rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as dr, 
+cume_dist() over (distribute by p_mfgr sort by p_mfgr, p_name) as cud, 
+first_value(p_size, true) over w1  as fv1
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2
following);
+
+from noop(on part 
+partition by p_mfgr 
+order by p_name) 
+INSERT OVERWRITE TABLE part_4 select p_mfgr, p_name, p_size, 
+rank() over (distribute by p_mfgr sort by p_name) as r, 
+dense_rank() over (distribute by p_mfgr sort by p_name) as dr, 
+sum(p_retailprice) over (distribute by p_mfgr sort by p_name rows between unbounded preceding
and current row)  as s  
+INSERT OVERWRITE TABLE part_5 select  p_mfgr,p_name, p_size,  
+round(sum(p_size) over (distribute by p_mfgr sort by p_size range between 5 preceding and
current row),1) as s2,
+rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as r, 
+dense_rank() over (distribute by p_mfgr sort by p_mfgr, p_name) as dr, 
+cume_dist() over (distribute by p_mfgr sort by p_mfgr, p_name) as cud, 
+first_value(p_size, true) over w1  as fv1
+window w1 as (distribute by p_mfgr sort by p_mfgr, p_name rows between 2 preceding and 2
following);
+
+select * from part_4;
+
+select * from part_5;
+
+-- 18. testMulti2OperatorsFunctionChainWithMap
+
+explain extended
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr,p_name) as r, 
+dense_rank() over (partition by p_mfgr,p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and
current row)  as s1
+from noop(on 
+        noopwithmap(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr 
+              order by p_mfgr) 
+            ) 
+          partition by p_mfgr,p_name 
+          order by p_mfgr,p_name) 
+        partition by p_mfgr,p_name  
+        order by p_mfgr,p_name) ;
+
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr,p_name) as r, 
+dense_rank() over (partition by p_mfgr,p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and
current row)  as s1
+from noop(on 
+        noopwithmap(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr 
+              order by p_mfgr) 
+            ) 
+          partition by p_mfgr,p_name 
+          order by p_mfgr,p_name) 
+        partition by p_mfgr,p_name  
+        order by p_mfgr,p_name) ;
+
+-- 19. testMulti3OperatorsFunctionChain
+
+explain extended
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row)  as s1
+from noop(on 
+        noop(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr 
+              order by p_mfgr) 
+            ) 
+          partition by p_mfgr,p_name 
+          order by p_mfgr,p_name) 
+        partition by p_mfgr  
+        order by p_mfgr ) ;
+        
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding
and current row)  as s1
+from noop(on 
+        noop(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr 
+              order by p_mfgr) 
+            ) 
+          partition by p_mfgr,p_name 
+          order by p_mfgr,p_name) 
+        partition by p_mfgr  
+        order by p_mfgr ) ;
+        
+-- 20. testMultiOperatorChainWithNoWindowing
+
+explain extended
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr order by p_name) as s1 
+from noop(on 
+        noop(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr,p_name 
+              order by p_mfgr,p_name) 
+            ) 
+          partition by p_mfgr 
+          order by p_mfgr));
+          
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr order by p_name) as s1 
+from noop(on 
+        noop(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr,p_name 
+              order by p_mfgr,p_name) 
+            ) 
+          partition by p_mfgr 
+          order by p_mfgr)); 
+
+
+-- 21. testMultiOperatorChainEndsWithNoopMap
+
+explain extended
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr,p_name) as r, 
+dense_rank() over (partition by p_mfgr,p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and
current row)  as s1 
+from noopwithmap(on 
+        noop(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr,p_name 
+              order by p_mfgr,p_name) 
+            ) 
+          partition by p_mfgr 
+          order by p_mfgr) 
+          partition by p_mfgr,p_name 
+          order by p_mfgr,p_name);
+          
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr,p_name) as r, 
+dense_rank() over (partition by p_mfgr,p_name) as dr, 
+p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and
current row)  as s1 
+from noopwithmap(on 
+        noop(on 
+          noop(on 
+              noop(on part 
+              partition by p_mfgr,p_name 
+              order by p_mfgr,p_name) 
+            ) 
+          partition by p_mfgr 
+          order by p_mfgr) 
+          partition by p_mfgr,p_name 
+          order by p_mfgr,p_name); 
+
+-- 22. testMultiOperatorChainWithDiffPartitionForWindow1
+
+explain extended
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr,p_name order by p_mfgr,p_name) as r, 
+dense_rank() over (partition by p_mfgr,p_name order by p_mfgr,p_name) as dr, 
+p_size, 
+sum(p_size) over (partition by p_mfgr,p_name order by p_mfgr,p_name rows between unbounded
preceding and current row) as s1, 
+sum(p_size) over (partition by p_mfgr,p_name order by p_mfgr,p_name rows between unbounded
preceding and current row)  as s2
+from noop(on 
+        noopwithmap(on 
+              noop(on part 
+              partition by p_mfgr, p_name 
+              order by p_mfgr, p_name) 
+          partition by p_mfgr 
+          order by p_mfgr
+          ));
+          
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr,p_name order by p_mfgr,p_name) as r, 
+dense_rank() over (partition by p_mfgr,p_name order by p_mfgr,p_name) as dr, 
+p_size, 
+sum(p_size) over (partition by p_mfgr,p_name order by p_mfgr,p_name rows between unbounded
preceding and current row) as s1, 
+sum(p_size) over (partition by p_mfgr,p_name order by p_mfgr,p_name rows between unbounded
preceding and current row)  as s2
+from noop(on 
+        noopwithmap(on 
+              noop(on part 
+              partition by p_mfgr, p_name 
+              order by p_mfgr, p_name) 
+          partition by p_mfgr 
+          order by p_mfgr
+          )); 
+
+-- 23. testMultiOperatorChainWithDiffPartitionForWindow2
+
+explain extended
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, 
+sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and
current row) as s1, 
+sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and
current row)  as s2
+from noopwithmap(on 
+        noop(on 
+              noop(on part 
+              partition by p_mfgr, p_name 
+              order by p_mfgr, p_name) 
+          ));
+          
+select p_mfgr, p_name,  
+rank() over (partition by p_mfgr order by p_name) as r, 
+dense_rank() over (partition by p_mfgr order by p_name) as dr, 
+p_size, 
+sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and
current row) as s1, 
+sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and
current row)  as s2
+from noopwithmap(on 
+        noop(on 
+              noop(on part 
+              partition by p_mfgr, p_name 
+              order by p_mfgr, p_name) 
+          ));
+ 

Added: hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out?rev=1610880&view=auto
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out (added) and hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
Tue Jul 15 23:09:31 2014 differ

Added: hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out?rev=1610880&view=auto
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out (added) and hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out
Tue Jul 15 23:09:31 2014 differ



Mime
View raw message