hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpull...@apache.org
Subject [51/51] [partial] hive git commit: HIVE-11110: Reorder applyPreJoinOrderingTransforms, add NotNULL/FilterMerge rules, improve Filter selectivity estimation (Laljo John Pullokkaran reviewed by Jesus Camacho Rodriguez, Ashutosh Chauhan)
Date Sat, 12 Dec 2015 06:55:40 GMT
HIVE-11110: Reorder applyPreJoinOrderingTransforms, add NotNULL/FilterMerge rules, improve Filter selectivity estimation (Laljo John Pullokkaran reviewed by Jesus Camacho Rodriguez, Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/08f73adc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/08f73adc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/08f73adc

Branch: refs/heads/master
Commit: 08f73adca5e7e189fac161d787997b0dfc017f7e
Parents: b187d42
Author: jpullokk <jpullokk@apache.org>
Authored: Fri Dec 11 22:43:43 2015 -0800
Committer: jpullokk <jpullokk@apache.org>
Committed: Fri Dec 11 22:43:43 2015 -0800

----------------------------------------------------------------------
 .../test/results/positive/hbase_queries.q.out   |   38 +-
 .../ql/optimizer/calcite/HiveCalciteUtil.java   |   98 +
 .../calcite/HiveDefaultRelMetadataProvider.java |    2 +
 .../ql/optimizer/calcite/RelOptHiveTable.java   |    9 +-
 .../hive/ql/optimizer/calcite/TraitsUtil.java   |    4 +
 .../calcite/reloperators/HiveProject.java       |   25 +-
 .../calcite/reloperators/HiveTableScan.java     |   46 +-
 .../rules/HiveFilterAggregateTransposeRule.java |   45 +
 .../rules/HiveFilterProjectTSTransposeRule.java |  113 +
 .../HiveJoinPushTransitivePredicatesRule.java   |   44 +-
 .../calcite/rules/HivePartitionPruneRule.java   |    2 +-
 .../calcite/rules/HivePreFilteringRule.java     |  170 +-
 .../stats/FilterSelectivityEstimator.java       |   48 +
 .../calcite/stats/HiveRelMdPredicates.java      |  645 ++++
 .../translator/SqlFunctionConverter.java        |    3 +-
 .../stats/annotation/StatsRulesProcFactory.java |   62 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |  117 +-
 .../hadoop/hive/ql/plan/ExprNodeDescUtils.java  |    8 +
 .../special_character_in_tabnames_1.q           |   26 +-
 .../bucket_mapjoin_mismatch1.q.out              |   28 +-
 .../sortmerge_mapjoin_mismatch_1.q.out          |   10 +-
 .../clientpositive/allcolref_in_udf.q.out       |   14 +-
 .../results/clientpositive/ambiguous_col.q.out  |   54 +-
 .../clientpositive/annotate_stats_join.q.out    |    8 +-
 .../annotate_stats_join_pkfk.q.out              |   61 +-
 .../archive_excludeHadoop20.q.out               |    1 -
 .../results/clientpositive/archive_multi.q.out  |    1 -
 .../results/clientpositive/auto_join1.q.out     |   14 +-
 .../results/clientpositive/auto_join10.q.out    |   12 +-
 .../results/clientpositive/auto_join12.q.out    |   66 +-
 .../results/clientpositive/auto_join13.q.out    |   10 +-
 .../results/clientpositive/auto_join15.q.out    |   12 +-
 .../results/clientpositive/auto_join16.q.out    |   18 +-
 .../results/clientpositive/auto_join17.q.out    |   14 +-
 .../results/clientpositive/auto_join19.q.out    |   14 +-
 .../results/clientpositive/auto_join2.q.out     |   24 +-
 .../results/clientpositive/auto_join22.q.out    |   18 +-
 .../results/clientpositive/auto_join24.q.out    |   10 +-
 .../results/clientpositive/auto_join26.q.out    |   20 +-
 .../results/clientpositive/auto_join3.q.out     |   18 +-
 .../results/clientpositive/auto_join30.q.out    |   64 +-
 .../results/clientpositive/auto_join33.q.out    |   16 +-
 .../results/clientpositive/auto_join4.q.out     |    4 +-
 .../results/clientpositive/auto_join5.q.out     |    4 +-
 .../results/clientpositive/auto_join8.q.out     |   18 +-
 .../results/clientpositive/auto_join9.q.out     |   14 +-
 .../auto_join_reordering_values.q.out           |   10 +-
 .../clientpositive/auto_join_stats.q.out        |  106 +-
 .../clientpositive/auto_join_stats2.q.out       |   46 +-
 .../auto_join_without_localtask.q.out           |  148 +-
 .../clientpositive/auto_smb_mapjoin_14.q.out    |   72 +-
 .../clientpositive/auto_sortmerge_join_1.q.out  |   20 +-
 .../clientpositive/auto_sortmerge_join_11.q.out |   22 +-
 .../clientpositive/auto_sortmerge_join_12.q.out |   12 +-
 .../clientpositive/auto_sortmerge_join_13.q.out |    6 +-
 .../clientpositive/auto_sortmerge_join_2.q.out  |   24 +-
 .../clientpositive/auto_sortmerge_join_3.q.out  |   28 +-
 .../clientpositive/auto_sortmerge_join_4.q.out  |   28 +-
 .../clientpositive/auto_sortmerge_join_5.q.out  |   28 +-
 .../clientpositive/auto_sortmerge_join_6.q.out  |  128 +-
 .../clientpositive/auto_sortmerge_join_7.q.out  |   28 +-
 .../clientpositive/auto_sortmerge_join_8.q.out  |   28 +-
 .../clientpositive/auto_sortmerge_join_9.q.out  |  204 +-
 .../clientpositive/bucket_map_join_spark1.q.out |   28 +-
 .../clientpositive/bucket_map_join_spark2.q.out |   28 +-
 .../clientpositive/bucket_map_join_spark3.q.out |   28 +-
 .../clientpositive/bucket_map_join_spark4.q.out |   40 +-
 .../clientpositive/bucketcontext_1.q.out        |    8 +-
 .../clientpositive/bucketcontext_2.q.out        |    8 +-
 .../clientpositive/bucketcontext_3.q.out        |    8 +-
 .../clientpositive/bucketcontext_4.q.out        |    8 +-
 .../clientpositive/bucketcontext_5.q.out        |    8 +-
 .../clientpositive/bucketcontext_6.q.out        |    8 +-
 .../clientpositive/bucketcontext_7.q.out        |    8 +-
 .../clientpositive/bucketcontext_8.q.out        |    8 +-
 .../bucketizedhiveinputformat.q.out             |    2 -
 .../results/clientpositive/bucketmapjoin1.q.out |   20 +-
 .../clientpositive/bucketmapjoin10.q.out        |    6 +-
 .../clientpositive/bucketmapjoin11.q.out        |   12 +-
 .../clientpositive/bucketmapjoin12.q.out        |   12 +-
 .../clientpositive/bucketmapjoin13.q.out        |   24 +-
 .../results/clientpositive/bucketmapjoin2.q.out |   30 +-
 .../results/clientpositive/bucketmapjoin3.q.out |   20 +-
 .../results/clientpositive/bucketmapjoin4.q.out |   20 +-
 .../results/clientpositive/bucketmapjoin5.q.out |   20 +-
 .../results/clientpositive/bucketmapjoin7.q.out |   12 +-
 .../results/clientpositive/bucketmapjoin8.q.out |   12 +-
 .../results/clientpositive/bucketmapjoin9.q.out |   12 +-
 .../clientpositive/bucketmapjoin_negative.q.out |   10 +-
 .../bucketmapjoin_negative2.q.out               |   10 +-
 .../bucketmapjoin_negative3.q.out               |   90 +-
 .../bucketsortoptimize_insert_2.q.out           |   56 +-
 .../bucketsortoptimize_insert_4.q.out           |   30 +-
 .../bucketsortoptimize_insert_5.q.out           |   40 +-
 .../bucketsortoptimize_insert_6.q.out           |  114 +-
 .../bucketsortoptimize_insert_7.q.out           |   12 +-
 .../bucketsortoptimize_insert_8.q.out           |    8 +-
 .../clientpositive/cbo_rp_auto_join1.q.out      |  396 ++-
 .../clientpositive/cbo_rp_auto_join17.q.out     |   14 +-
 .../cbo_rp_cross_product_check_2.q.out          |   56 +-
 .../clientpositive/cbo_rp_lineage2.q.out        |    8 +-
 .../results/clientpositive/cbo_udf_max.q.out    |    4 +-
 .../clientpositive/column_access_stats.q.out    |   52 +-
 .../clientpositive/constprog_partitioner.q.out  |   44 +-
 .../clientpositive/correlationoptimizer1.q.out  |  190 +-
 .../clientpositive/correlationoptimizer10.q.out |  176 +-
 .../clientpositive/correlationoptimizer11.q.out |   72 +-
 .../clientpositive/correlationoptimizer12.q.out |   16 +-
 .../clientpositive/correlationoptimizer13.q.out |   42 +-
 .../clientpositive/correlationoptimizer14.q.out |  234 +-
 .../clientpositive/correlationoptimizer15.q.out |   60 +-
 .../clientpositive/correlationoptimizer2.q.out  |  104 +-
 .../clientpositive/correlationoptimizer3.q.out  |  220 +-
 .../clientpositive/correlationoptimizer4.q.out  |   74 +-
 .../clientpositive/correlationoptimizer5.q.out  |  110 +-
 .../clientpositive/correlationoptimizer6.q.out  |  616 ++--
 .../clientpositive/correlationoptimizer7.q.out  |  104 +-
 .../clientpositive/correlationoptimizer8.q.out  |   34 +-
 .../clientpositive/correlationoptimizer9.q.out  |  120 +-
 .../results/clientpositive/cross_join.q.out     |   28 +-
 .../clientpositive/cross_product_check_1.q.out  |   66 +-
 .../clientpositive/cross_product_check_2.q.out  |   54 +-
 .../results/clientpositive/decimal_join2.q.out  |   36 +-
 .../results/clientpositive/decimal_udf.q.out    |   12 +-
 .../clientpositive/dynamic_rdd_cache.q.out      |   34 +-
 .../encryption_join_unencrypted_tbl.q.out       |   20 +-
 ...on_join_with_different_encryption_keys.q.out |   16 +-
 .../clientpositive/explain_logical.q.out        |  198 +-
 .../clientpositive/filter_cond_pushdown.q.out   |  241 +-
 .../clientpositive/filter_join_breaktask.q.out  |   32 +-
 .../clientpositive/groupby_join_pushdown.q.out  |  320 +-
 .../groupby_multi_single_reducer3.q.out         |   16 +-
 .../clientpositive/groupby_position.q.out       |   30 +-
 .../clientpositive/groupby_sort_1_23.q.out      |   46 +-
 .../clientpositive/groupby_sort_skew_1_23.q.out |   50 +-
 .../test/results/clientpositive/having2.q.out   |   66 +-
 .../identity_project_remove_skip.q.out          |   10 +-
 .../clientpositive/index_auto_mult_tables.q.out |   54 +-
 .../index_auto_mult_tables_compact.q.out        |   42 +-
 .../clientpositive/index_auto_self_join.q.out   |   58 +-
 .../results/clientpositive/index_bitmap3.q.out  |   30 +-
 .../clientpositive/index_bitmap_auto.q.out      |   30 +-
 .../index_bitmap_auto_partitioned.q.out         |    6 +-
 .../index_bitmap_compression.q.out              |    6 +-
 .../infer_bucket_sort_map_operators.q.out       |   22 +-
 .../test/results/clientpositive/innerjoin.q.out |   18 +-
 .../test/results/clientpositive/input39.q.out   |   10 +-
 .../results/clientpositive/input_part9.q.out    |    4 +-
 .../clientpositive/input_testxpath2.q.out       |    6 +-
 .../clientpositive/input_testxpath4.q.out       |   12 +-
 ql/src/test/results/clientpositive/join1.q.out  |   18 +-
 ql/src/test/results/clientpositive/join10.q.out |   18 +-
 ql/src/test/results/clientpositive/join12.q.out |   65 +-
 ql/src/test/results/clientpositive/join13.q.out |   16 +-
 ql/src/test/results/clientpositive/join15.q.out |   18 +-
 ql/src/test/results/clientpositive/join16.q.out |   20 +-
 ql/src/test/results/clientpositive/join17.q.out |   18 +-
 ql/src/test/results/clientpositive/join2.q.out  |   32 +-
 ql/src/test/results/clientpositive/join22.q.out |   28 +-
 ql/src/test/results/clientpositive/join25.q.out |   10 +-
 ql/src/test/results/clientpositive/join26.q.out |   12 +-
 ql/src/test/results/clientpositive/join27.q.out |   10 +-
 ql/src/test/results/clientpositive/join28.q.out |   22 +-
 ql/src/test/results/clientpositive/join29.q.out |   42 +-
 ql/src/test/results/clientpositive/join3.q.out  |   24 +-
 ql/src/test/results/clientpositive/join30.q.out |   16 +-
 ql/src/test/results/clientpositive/join31.q.out |   40 +-
 ql/src/test/results/clientpositive/join32.q.out |   22 +-
 .../clientpositive/join32_lessSize.q.out        |  124 +-
 ql/src/test/results/clientpositive/join33.q.out |   22 +-
 ql/src/test/results/clientpositive/join34.q.out |    4 +-
 ql/src/test/results/clientpositive/join35.q.out |    4 +-
 ql/src/test/results/clientpositive/join36.q.out |   10 +-
 ql/src/test/results/clientpositive/join37.q.out |   10 +-
 ql/src/test/results/clientpositive/join40.q.out |   32 +-
 ql/src/test/results/clientpositive/join42.q.out |   77 +-
 ql/src/test/results/clientpositive/join43.q.out |   66 +-
 ql/src/test/results/clientpositive/join8.q.out  |   16 +-
 ql/src/test/results/clientpositive/join9.q.out  |   18 +-
 .../clientpositive/join_alt_syntax.q.out        |  144 +-
 .../clientpositive/join_cond_pushdown_1.q.out   |   86 +-
 .../clientpositive/join_cond_pushdown_2.q.out   |   74 +-
 .../clientpositive/join_cond_pushdown_3.q.out   |   86 +-
 .../clientpositive/join_cond_pushdown_4.q.out   |   74 +-
 .../join_cond_pushdown_unqual1.q.out            |   24 +-
 .../join_cond_pushdown_unqual2.q.out            |   44 +-
 .../join_cond_pushdown_unqual3.q.out            |   24 +-
 .../join_cond_pushdown_unqual4.q.out            |   44 +-
 .../clientpositive/join_grp_diff_keys.q.out     |    8 +-
 .../results/clientpositive/join_map_ppr.q.out   |   24 +-
 .../join_merge_multi_expressions.q.out          |   20 +-
 .../results/clientpositive/join_nullsafe.q.out  |   36 +-
 .../clientpositive/join_on_varchar.q.out        |   14 +-
 .../results/clientpositive/join_parse.q.out     |  130 +-
 .../test/results/clientpositive/join_rc.q.out   |   18 +-
 .../results/clientpositive/join_reorder.q.out   |   24 +-
 .../test/results/clientpositive/join_star.q.out |   54 +-
 .../results/clientpositive/join_thrift.q.out    |   14 +-
 .../test/results/clientpositive/join_vc.q.out   |   30 +-
 .../clientpositive/lateral_view_cp.q.out        |   24 +-
 .../results/clientpositive/limit_pushdown.q.out |   14 +-
 .../limit_pushdown_negative.q.out               |   14 +-
 .../test/results/clientpositive/lineage2.q.out  |   18 +-
 .../test/results/clientpositive/lineage3.q.out  |   20 +-
 .../list_bucket_query_oneskew_2.q.out           |    2 +-
 .../llap/bucket_map_join_tez1.q.out             |  341 +-
 .../llap/bucket_map_join_tez2.q.out             |  135 +-
 .../llap/dynamic_partition_pruning.q.out        |  269 +-
 .../llap/dynamic_partition_pruning_2.q.out      |   21 +-
 .../llap/hybridgrace_hashjoin_1.q.out           |   97 +-
 .../llap/hybridgrace_hashjoin_2.q.out           |  128 +-
 .../clientpositive/llap/lvj_mapjoin.q.out       |   56 +-
 .../clientpositive/llap/mapjoin_decimal.q.out   |   18 +-
 .../results/clientpositive/llap/mergejoin.q.out | 1614 +++++++--
 .../test/results/clientpositive/llap/mrr.q.out  |  130 +-
 .../llap/tez_bmj_schema_evolution.q.out         |   18 +-
 .../llap/tez_dynpart_hashjoin_1.q.out           |  120 +-
 .../llap/tez_dynpart_hashjoin_2.q.out           |  118 +-
 .../clientpositive/llap/tez_join_hash.q.out     |  145 +-
 .../llap/tez_join_result_complex.q.out          |   64 +-
 .../clientpositive/llap/tez_self_join.q.out     |   28 +-
 .../results/clientpositive/llap/tez_smb_1.q.out |   64 +-
 .../clientpositive/llap/tez_smb_main.q.out      |  207 +-
 .../results/clientpositive/llap/tez_union.q.out |  172 +-
 .../llap/tez_vector_dynpart_hashjoin_1.q.out    |  120 +-
 .../llap/tez_vector_dynpart_hashjoin_2.q.out    |  122 +-
 .../vectorized_dynamic_partition_pruning.q.out  |  269 +-
 .../llap/vectorized_nested_mapjoin.q.out        |   66 +-
 .../clientpositive/llap_uncompressed.q.out      |    6 +-
 .../clientpositive/louter_join_ppr.q.out        |   28 +-
 .../clientpositive/mapjoin_distinct.q.out       |   60 +-
 .../clientpositive/mapjoin_mapjoin.q.out        |   60 +-
 .../clientpositive/mapjoin_memcheck.q.out       |   12 +-
 .../clientpositive/mapjoin_subquery.q.out       |   44 +-
 .../test/results/clientpositive/mergejoin.q.out |  210 +-
 .../results/clientpositive/mergejoins.q.out     |   38 +-
 .../clientpositive/mergejoins_mixed.q.out       |  266 +-
 .../results/clientpositive/metadataonly1.q.out  |    8 +-
 .../results/clientpositive/multiMapJoin1.q.out  |  384 +--
 .../results/clientpositive/multiMapJoin2.q.out  |  324 +-
 .../clientpositive/multi_join_union.q.out       |   26 +-
 .../results/clientpositive/nonmr_fetch.q.out    |   16 +-
 .../offset_limit_ppd_optimizer.q.out            |   10 +-
 .../test/results/clientpositive/orc_llap.q.out  |   52 +-
 .../clientpositive/orc_predicate_pushdown.q.out |   32 +-
 .../results/clientpositive/parallel_join1.q.out |   18 +-
 .../results/clientpositive/parquet_join.q.out   |   36 +-
 .../parquet_predicate_pushdown.q.out            |   32 +-
 ql/src/test/results/clientpositive/pcr.q.out    |   40 +-
 .../results/clientpositive/pointlookup2.q.out   |   40 +-
 .../results/clientpositive/pointlookup3.q.out   |   40 +-
 .../test/results/clientpositive/ppd_gby.q.out   |   49 +-
 .../test/results/clientpositive/ppd_gby2.q.out  |   55 +-
 .../results/clientpositive/ppd_gby_join.q.out   |   96 +-
 .../test/results/clientpositive/ppd_join.q.out  |   86 +-
 .../test/results/clientpositive/ppd_join2.q.out |  133 +-
 .../test/results/clientpositive/ppd_join3.q.out |  155 +-
 .../test/results/clientpositive/ppd_join5.q.out |   10 +-
 .../clientpositive/ppd_join_filter.q.out        |  130 +-
 .../clientpositive/ppd_multi_insert.q.out       |   68 +-
 .../clientpositive/ppd_outer_join2.q.out        |   66 +-
 .../clientpositive/ppd_outer_join3.q.out        |   66 +-
 .../clientpositive/ppd_outer_join4.q.out        |  223 +-
 .../clientpositive/ppd_outer_join5.q.out        |  231 +-
 .../results/clientpositive/ppd_random.q.out     |   16 +-
 .../clientpositive/ppd_repeated_alias.q.out     |    2 +-
 .../results/clientpositive/ppd_udf_case.q.out   |   59 +-
 .../results/clientpositive/ppd_udf_col.q.out    |  178 +-
 .../test/results/clientpositive/ppd_union.q.out |   42 +-
 .../results/clientpositive/ppd_union_view.q.out |   34 +-
 ql/src/test/results/clientpositive/ppd_vc.q.out |   30 +-
 ql/src/test/results/clientpositive/ptf.q.out    |   72 +-
 .../results/clientpositive/ptf_streaming.q.out  |   58 +-
 .../results/clientpositive/ptfgroupbyjoin.q.out |   44 +-
 .../results/clientpositive/quotedid_basic.q.out |   12 +-
 .../clientpositive/quotedid_partition.q.out     |    4 +-
 .../reduce_deduplicate_exclude_join.q.out       |   12 +-
 .../reduce_deduplicate_extended.q.out           |   86 +-
 .../test/results/clientpositive/regex_col.q.out |   22 +-
 .../clientpositive/router_join_ppr.q.out        |   30 +-
 .../runtime_skewjoin_mapjoin_spark.q.out        |   58 +-
 .../test/results/clientpositive/sample8.q.out   |   46 +-
 .../clientpositive/select_transform_hint.q.out  |   24 +-
 .../test/results/clientpositive/semijoin.q.out  |  272 +-
 .../test/results/clientpositive/skewjoin.q.out  |  259 +-
 .../clientpositive/skewjoin_mapjoin10.q.out     |   32 +-
 .../clientpositive/skewjoin_mapjoin6.q.out      |   48 +-
 .../clientpositive/skewjoin_noskew.q.out        |   18 +-
 .../clientpositive/skewjoin_onesideskew.q.out   |   18 +-
 .../results/clientpositive/skewjoinopt10.q.out  |   52 +-
 .../results/clientpositive/skewjoinopt12.q.out  |    8 +-
 .../results/clientpositive/skewjoinopt15.q.out  |   40 +-
 .../results/clientpositive/skewjoinopt16.q.out  |    8 +-
 .../results/clientpositive/skewjoinopt17.q.out  |    8 +-
 .../results/clientpositive/skewjoinopt18.q.out  |   14 +-
 .../results/clientpositive/skewjoinopt2.q.out   |   16 +-
 .../results/clientpositive/smb_mapjoin9.q.out   |    2 +-
 .../results/clientpositive/smb_mapjoin_1.q.out  |    2 +-
 .../results/clientpositive/smb_mapjoin_10.q.out |    2 +-
 .../results/clientpositive/smb_mapjoin_11.q.out |    2 +-
 .../results/clientpositive/smb_mapjoin_12.q.out |    4 +-
 .../results/clientpositive/smb_mapjoin_13.q.out |   14 +-
 .../results/clientpositive/smb_mapjoin_14.q.out |   18 +-
 .../results/clientpositive/smb_mapjoin_15.q.out |   18 +-
 .../results/clientpositive/smb_mapjoin_16.q.out |    2 +-
 .../results/clientpositive/smb_mapjoin_17.q.out |    2 +-
 .../results/clientpositive/smb_mapjoin_2.q.out  |    4 +-
 .../results/clientpositive/smb_mapjoin_25.q.out |  105 +-
 .../results/clientpositive/smb_mapjoin_3.q.out  |    2 +-
 .../results/clientpositive/smb_mapjoin_4.q.out  |    2 +-
 .../results/clientpositive/smb_mapjoin_6.q.out  |    4 +-
 .../clientpositive/sort_merge_join_desc_2.q.out |    2 +-
 .../clientpositive/sort_merge_join_desc_3.q.out |    2 +-
 .../clientpositive/sort_merge_join_desc_4.q.out |    6 +-
 .../clientpositive/sort_merge_join_desc_5.q.out |    2 +-
 .../clientpositive/sort_merge_join_desc_6.q.out |    6 +-
 .../clientpositive/sort_merge_join_desc_7.q.out |    6 +-
 .../clientpositive/sort_merge_join_desc_8.q.out |    6 +-
 .../spark/annotate_stats_join.q.out             |    8 +-
 .../clientpositive/spark/auto_join1.q.out       |   14 +-
 .../clientpositive/spark/auto_join10.q.out      |   12 +-
 .../clientpositive/spark/auto_join12.q.out      |   73 +-
 .../clientpositive/spark/auto_join13.q.out      |   10 +-
 .../clientpositive/spark/auto_join15.q.out      |   12 +-
 .../clientpositive/spark/auto_join16.q.out      |   14 +-
 .../clientpositive/spark/auto_join17.q.out      |   14 +-
 .../clientpositive/spark/auto_join19.q.out      |   14 +-
 .../clientpositive/spark/auto_join2.q.out       |   98 +-
 .../clientpositive/spark/auto_join22.q.out      |   18 +-
 .../clientpositive/spark/auto_join24.q.out      |   10 +-
 .../clientpositive/spark/auto_join26.q.out      |   20 +-
 .../clientpositive/spark/auto_join3.q.out       |   18 +-
 .../clientpositive/spark/auto_join30.q.out      |   38 +-
 .../clientpositive/spark/auto_join8.q.out       |   14 +-
 .../clientpositive/spark/auto_join9.q.out       |   14 +-
 .../spark/auto_join_reordering_values.q.out     |   10 +-
 .../clientpositive/spark/auto_join_stats.q.out  |   58 +-
 .../clientpositive/spark/auto_join_stats2.q.out |   46 +-
 .../spark/auto_join_without_localtask.q.out     |   68 +-
 .../spark/auto_smb_mapjoin_14.q.out             |  126 +-
 .../spark/auto_sortmerge_join_1.q.out           |   18 +-
 .../spark/auto_sortmerge_join_12.q.out          |   12 +-
 .../spark/auto_sortmerge_join_13.q.out          |   44 +-
 .../spark/auto_sortmerge_join_2.q.out           |   16 +-
 .../spark/auto_sortmerge_join_3.q.out           |   22 +-
 .../spark/auto_sortmerge_join_4.q.out           |   22 +-
 .../spark/auto_sortmerge_join_5.q.out           |   22 +-
 .../spark/auto_sortmerge_join_6.q.out           |  144 +-
 .../spark/auto_sortmerge_join_7.q.out           |   22 +-
 .../spark/auto_sortmerge_join_8.q.out           |   22 +-
 .../spark/auto_sortmerge_join_9.q.out           |  220 +-
 .../spark/bucket_map_join_spark1.q.out          |   28 +-
 .../spark/bucket_map_join_spark2.q.out          |   28 +-
 .../spark/bucket_map_join_spark3.q.out          |   28 +-
 .../spark/bucket_map_join_spark4.q.out          |  140 +-
 .../spark/bucket_map_join_tez1.q.out            |  361 +-
 .../spark/bucket_map_join_tez2.q.out            |  185 +-
 .../spark/bucketizedhiveinputformat.q.out       |    2 -
 .../clientpositive/spark/bucketmapjoin1.q.out   |   20 +-
 .../clientpositive/spark/bucketmapjoin10.q.out  |    6 +-
 .../clientpositive/spark/bucketmapjoin11.q.out  |   12 +-
 .../clientpositive/spark/bucketmapjoin12.q.out  |   12 +-
 .../clientpositive/spark/bucketmapjoin13.q.out  |   24 +-
 .../clientpositive/spark/bucketmapjoin2.q.out   |   30 +-
 .../clientpositive/spark/bucketmapjoin3.q.out   |   20 +-
 .../clientpositive/spark/bucketmapjoin4.q.out   |   20 +-
 .../clientpositive/spark/bucketmapjoin5.q.out   |   20 +-
 .../clientpositive/spark/bucketmapjoin7.q.out   |   12 +-
 .../clientpositive/spark/bucketmapjoin8.q.out   |   12 +-
 .../clientpositive/spark/bucketmapjoin9.q.out   |   12 +-
 .../spark/bucketmapjoin_negative.q.out          |   10 +-
 .../spark/bucketmapjoin_negative2.q.out         |   10 +-
 .../spark/bucketmapjoin_negative3.q.out         |   90 +-
 .../spark/bucketsortoptimize_insert_2.q.out     |  108 +-
 .../spark/bucketsortoptimize_insert_4.q.out     |   30 +-
 .../spark/bucketsortoptimize_insert_6.q.out     |  154 +-
 .../spark/bucketsortoptimize_insert_7.q.out     |   54 +-
 .../spark/bucketsortoptimize_insert_8.q.out     |   36 +-
 .../spark/column_access_stats.q.out             |   52 +-
 .../spark/constprog_partitioner.q.out           |   44 +-
 .../clientpositive/spark/cross_join.q.out       |   28 +-
 .../spark/cross_product_check_1.q.out           |   66 +-
 .../spark/cross_product_check_2.q.out           |   48 +-
 .../spark/dynamic_rdd_cache.q.out               |   34 +-
 .../spark/filter_join_breaktask.q.out           |   28 +-
 .../spark/groupby_multi_single_reducer3.q.out   |   16 +-
 .../clientpositive/spark/groupby_position.q.out |   32 +-
 .../spark/groupby_sort_1_23.q.out               |   46 +-
 .../spark/groupby_sort_skew_1_23.q.out          |   50 +-
 .../spark/identity_project_remove_skip.q.out    |    8 +-
 .../spark/index_auto_self_join.q.out            |   42 +-
 .../clientpositive/spark/index_bitmap3.q.out    |   30 +-
 .../spark/index_bitmap_auto.q.out               |   30 +-
 .../spark/infer_bucket_sort_map_operators.q.out |   38 +-
 .../infer_bucket_sort_reducers_power_two.q.out  |    8 +-
 .../clientpositive/spark/innerjoin.q.out        |   18 +-
 .../results/clientpositive/spark/join1.q.out    |   18 +-
 .../results/clientpositive/spark/join10.q.out   |   18 +-
 .../results/clientpositive/spark/join12.q.out   |   51 +-
 .../results/clientpositive/spark/join13.q.out   |   16 +-
 .../results/clientpositive/spark/join15.q.out   |   18 +-
 .../results/clientpositive/spark/join16.q.out   |   20 +-
 .../results/clientpositive/spark/join17.q.out   |   18 +-
 .../results/clientpositive/spark/join2.q.out    |   32 +-
 .../results/clientpositive/spark/join22.q.out   |   22 +-
 .../results/clientpositive/spark/join25.q.out   |   10 +-
 .../results/clientpositive/spark/join26.q.out   |   12 +-
 .../results/clientpositive/spark/join27.q.out   |   10 +-
 .../results/clientpositive/spark/join28.q.out   |   22 +-
 .../results/clientpositive/spark/join29.q.out   |   26 +-
 .../results/clientpositive/spark/join3.q.out    |   24 +-
 .../results/clientpositive/spark/join30.q.out   |   16 +-
 .../results/clientpositive/spark/join31.q.out   |   28 +-
 .../results/clientpositive/spark/join32.q.out   |   22 +-
 .../clientpositive/spark/join32_lessSize.q.out  |  138 +-
 .../results/clientpositive/spark/join33.q.out   |   22 +-
 .../results/clientpositive/spark/join34.q.out   |    6 +-
 .../results/clientpositive/spark/join35.q.out   |    6 +-
 .../results/clientpositive/spark/join36.q.out   |   10 +-
 .../results/clientpositive/spark/join37.q.out   |   10 +-
 .../results/clientpositive/spark/join40.q.out   |   32 +-
 .../results/clientpositive/spark/join8.q.out    |   16 +-
 .../results/clientpositive/spark/join9.q.out    |   18 +-
 .../clientpositive/spark/join_alt_syntax.q.out  |  144 +-
 .../spark/join_cond_pushdown_1.q.out            |   86 +-
 .../spark/join_cond_pushdown_2.q.out            |   74 +-
 .../spark/join_cond_pushdown_3.q.out            |   86 +-
 .../spark/join_cond_pushdown_4.q.out            |   74 +-
 .../spark/join_cond_pushdown_unqual1.q.out      |   24 +-
 .../spark/join_cond_pushdown_unqual2.q.out      |   44 +-
 .../spark/join_cond_pushdown_unqual3.q.out      |   24 +-
 .../spark/join_cond_pushdown_unqual4.q.out      |   44 +-
 .../clientpositive/spark/join_map_ppr.q.out     |   24 +-
 .../spark/join_merge_multi_expressions.q.out    |   20 +-
 .../clientpositive/spark/join_nullsafe.q.out    |   36 +-
 .../results/clientpositive/spark/join_rc.q.out  |   18 +-
 .../clientpositive/spark/join_reorder.q.out     |   24 +-
 .../clientpositive/spark/join_star.q.out        |   54 +-
 .../clientpositive/spark/join_thrift.q.out      |   14 +-
 .../results/clientpositive/spark/join_vc.q.out  |   30 +-
 .../clientpositive/spark/limit_pushdown.q.out   |   14 +-
 .../clientpositive/spark/louter_join_ppr.q.out  |   18 +-
 .../clientpositive/spark/mapjoin_decimal.q.out  |   16 +-
 .../clientpositive/spark/mapjoin_distinct.q.out |   60 +-
 .../clientpositive/spark/mapjoin_mapjoin.q.out  |  158 +-
 .../clientpositive/spark/mapjoin_memcheck.q.out |   12 +-
 .../clientpositive/spark/mapjoin_subquery.q.out |   44 +-
 .../clientpositive/spark/mergejoins.q.out       |   38 +-
 .../clientpositive/spark/mergejoins_mixed.q.out |  266 +-
 .../clientpositive/spark/multi_join_union.q.out |   28 +-
 .../clientpositive/spark/parallel_join1.q.out   |   18 +-
 .../clientpositive/spark/parquet_join.q.out     |   42 +-
 .../test/results/clientpositive/spark/pcr.q.out |   40 +-
 .../clientpositive/spark/ppd_gby_join.q.out     |   96 +-
 .../results/clientpositive/spark/ppd_join.q.out |   86 +-
 .../clientpositive/spark/ppd_join2.q.out        |  133 +-
 .../clientpositive/spark/ppd_join3.q.out        |  153 +-
 .../clientpositive/spark/ppd_join5.q.out        |   10 +-
 .../clientpositive/spark/ppd_join_filter.q.out  |  130 +-
 .../clientpositive/spark/ppd_multi_insert.q.out |   68 +-
 .../clientpositive/spark/ppd_outer_join2.q.out  |   66 +-
 .../clientpositive/spark/ppd_outer_join3.q.out  |   66 +-
 .../clientpositive/spark/ppd_outer_join4.q.out  |  211 +-
 .../clientpositive/spark/ppd_outer_join5.q.out  |  207 +-
 .../test/results/clientpositive/spark/ptf.q.out |   72 +-
 .../clientpositive/spark/ptf_streaming.q.out    |   58 +-
 .../spark/reduce_deduplicate_exclude_join.q.out |   12 +-
 .../clientpositive/spark/router_join_ppr.q.out  |   20 +-
 .../spark/runtime_skewjoin_mapjoin_spark.q.out  |   26 +-
 .../results/clientpositive/spark/sample8.q.out  |   46 +-
 .../results/clientpositive/spark/semijoin.q.out |  272 +-
 .../results/clientpositive/spark/skewjoin.q.out |  283 +-
 .../clientpositive/spark/skewjoin_noskew.q.out  |   18 +-
 .../clientpositive/spark/skewjoinopt10.q.out    |   48 +-
 .../clientpositive/spark/skewjoinopt12.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt15.q.out    |   32 +-
 .../clientpositive/spark/skewjoinopt16.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt17.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt18.q.out    |   14 +-
 .../clientpositive/spark/skewjoinopt2.q.out     |   16 +-
 .../clientpositive/spark/smb_mapjoin_1.q.out    |   14 +-
 .../clientpositive/spark/smb_mapjoin_10.q.out   |    8 +-
 .../clientpositive/spark/smb_mapjoin_11.q.out   |   12 +-
 .../clientpositive/spark/smb_mapjoin_12.q.out   |   24 +-
 .../clientpositive/spark/smb_mapjoin_13.q.out   |   22 +-
 .../clientpositive/spark/smb_mapjoin_14.q.out   |   52 +-
 .../clientpositive/spark/smb_mapjoin_15.q.out   |   42 +-
 .../clientpositive/spark/smb_mapjoin_16.q.out   |    4 +-
 .../clientpositive/spark/smb_mapjoin_17.q.out   |    4 +-
 .../clientpositive/spark/smb_mapjoin_2.q.out    |   16 +-
 .../clientpositive/spark/smb_mapjoin_25.q.out   |   83 +-
 .../clientpositive/spark/smb_mapjoin_3.q.out    |   14 +-
 .../clientpositive/spark/smb_mapjoin_4.q.out    |    8 +-
 .../clientpositive/spark/smb_mapjoin_5.q.out    |    6 +-
 .../clientpositive/spark/smb_mapjoin_6.q.out    |   16 +-
 .../clientpositive/spark/subquery_exists.q.out  |   22 +-
 .../clientpositive/spark/subquery_in.q.out      |  145 +-
 .../spark/subquery_multiinsert.q.java1.7.out    |   14 +-
 .../spark/table_access_keys_stats.q.out         |   37 +-
 .../clientpositive/spark/temp_table_join1.q.out |   54 +-
 .../results/clientpositive/spark/union26.q.out  |   28 +-
 .../results/clientpositive/spark/union32.q.out  |   80 +-
 .../results/clientpositive/spark/union34.q.out  |   44 +-
 .../clientpositive/spark/union_top_level.q.out  |   16 +-
 .../spark/vector_decimal_mapjoin.q.out          |   12 +-
 .../spark/vector_mapjoin_reduce.q.out           |  166 +-
 .../spark/vectorization_short_regress.q.out     |   18 +-
 .../spark/vectorized_mapjoin.q.out              |   12 +-
 .../spark/vectorized_nested_mapjoin.q.out       |   32 +-
 .../clientpositive/spark/vectorized_ptf.q.out   |   72 +-
 .../spark/vectorized_shufflejoin.q.out          |   16 +-
 .../test/results/clientpositive/stats11.q.out   |   20 +-
 .../subq_where_serialization.q.out              |   30 +-
 .../clientpositive/subquery_exists.q.out        |   22 +-
 .../clientpositive/subquery_exists_having.q.out |   44 +-
 .../results/clientpositive/subquery_in.q.out    |  145 +-
 .../clientpositive/subquery_in_having.q.out     |  118 +-
 .../subquery_multiinsert.q.java1.7.out          |   14 +-
 .../results/clientpositive/subquery_notin.q.out |   16 +-
 .../subquery_notin_having.q.java1.7.out         |    6 +-
 .../subquery_unqual_corr_expr.q.out             |   16 +-
 .../subquery_unqualcolumnrefs.q.out             |  142 +-
 .../results/clientpositive/subquery_views.q.out |   26 +-
 .../table_access_keys_stats.q.out               |   37 +-
 .../clientpositive/temp_table_join1.q.out       |   54 +-
 .../results/clientpositive/tez/auto_join1.q.out |   16 +-
 .../clientpositive/tez/auto_join30.q.out        |   50 +-
 .../tez/auto_sortmerge_join_1.q.out             |   18 +-
 .../tez/auto_sortmerge_join_11.q.out            |   26 +-
 .../tez/auto_sortmerge_join_12.q.out            |   12 +-
 .../tez/auto_sortmerge_join_13.q.out            |   54 +-
 .../tez/auto_sortmerge_join_2.q.out             |   28 +-
 .../tez/auto_sortmerge_join_3.q.out             |   42 +-
 .../tez/auto_sortmerge_join_4.q.out             |   42 +-
 .../tez/auto_sortmerge_join_5.q.out             |   34 +-
 .../tez/auto_sortmerge_join_6.q.out             |  192 +-
 .../tez/auto_sortmerge_join_7.q.out             |   42 +-
 .../tez/auto_sortmerge_join_8.q.out             |   42 +-
 .../tez/auto_sortmerge_join_9.q.out             |  268 +-
 .../tez/bucket_map_join_tez1.q.out              |  339 +-
 .../tez/bucket_map_join_tez2.q.out              |  133 +-
 .../tez/correlationoptimizer1.q.out             |  208 +-
 .../results/clientpositive/tez/cross_join.q.out |   30 +-
 .../tez/cross_product_check_1.q.out             |   66 +-
 .../tez/cross_product_check_2.q.out             |   54 +-
 .../tez/dynamic_partition_pruning.q.out         |  228 +-
 .../tez/dynamic_partition_pruning_2.q.out       |   21 +-
 .../clientpositive/tez/explainuser_1.q.out      | 1502 ++++----
 .../clientpositive/tez/explainuser_2.q.out      | 3195 +++++++++---------
 .../clientpositive/tez/explainuser_3.q.out      |   34 +-
 .../tez/filter_join_breaktask.q.out             |   28 +-
 .../tez/hybridgrace_hashjoin_1.q.out            |   97 +-
 .../tez/hybridgrace_hashjoin_2.q.out            |  128 +-
 .../test/results/clientpositive/tez/join1.q.out |   18 +-
 .../clientpositive/tez/join_nullsafe.q.out      |   36 +-
 .../clientpositive/tez/limit_pushdown.q.out     |   14 +-
 .../clientpositive/tez/lvj_mapjoin.q.out        |   56 +-
 .../clientpositive/tez/mapjoin_decimal.q.out    |   18 +-
 .../clientpositive/tez/mapjoin_mapjoin.q.out    |   74 +-
 .../results/clientpositive/tez/mergejoin.q.out  |  204 +-
 .../test/results/clientpositive/tez/mrr.q.out   |  130 +-
 .../test/results/clientpositive/tez/ptf.q.out   |   72 +-
 .../clientpositive/tez/ptf_streaming.q.out      |   58 +-
 .../results/clientpositive/tez/skewjoin.q.out   |  109 +-
 .../clientpositive/tez/subquery_exists.q.out    |   22 +-
 .../clientpositive/tez/subquery_in.q.out        |  145 +-
 .../tez/tez_bmj_schema_evolution.q.out          |   18 +-
 .../tez/tez_dynpart_hashjoin_1.q.out            |  120 +-
 .../tez/tez_dynpart_hashjoin_2.q.out            |  118 +-
 .../clientpositive/tez/tez_join_hash.q.out      |  144 +-
 .../tez/tez_join_result_complex.q.out           |   64 +-
 .../clientpositive/tez/tez_self_join.q.out      |   28 +-
 .../results/clientpositive/tez/tez_smb_1.q.out  |   64 +-
 .../clientpositive/tez/tez_smb_empty.q.out      |  116 +-
 .../clientpositive/tez/tez_smb_main.q.out       |  206 +-
 .../results/clientpositive/tez/tez_union.q.out  |  172 +-
 .../tez/tez_vector_dynpart_hashjoin_1.q.out     |  120 +-
 .../tez/tez_vector_dynpart_hashjoin_2.q.out     |  120 +-
 .../clientpositive/tez/unionDistinct_1.q.out    |  190 +-
 .../tez/vector_auto_smb_mapjoin_14.q.out        |  300 +-
 .../tez/vector_binary_join_groupby.q.out        |   14 +-
 .../tez/vector_char_mapjoin1.q.out              |   56 +-
 .../tez/vector_decimal_expressions.q.out        |    8 +-
 .../tez/vector_decimal_mapjoin.q.out            |   14 +-
 .../clientpositive/tez/vector_decimal_udf.q.out |   18 +-
 .../clientpositive/tez/vector_if_expr.q.out     |   10 +-
 .../tez/vector_interval_mapjoin.q.out           |   24 +-
 .../clientpositive/tez/vector_join30.q.out      |   50 +-
 .../tez/vector_join_part_col_char.q.out         |   22 +-
 .../tez/vector_leftsemi_mapjoin.q.out           | 1500 ++++----
 .../tez/vector_mapjoin_reduce.q.out             |  133 +-
 .../tez/vector_nullsafe_join.q.out              |   64 +-
 .../tez/vector_reduce_groupby_decimal.q.out     |   12 +-
 .../tez/vector_varchar_mapjoin1.q.out           |   56 +-
 .../clientpositive/tez/vectorization_8.q.out    |   16 +-
 .../tez/vectorization_limit.q.out               |   22 +-
 .../tez/vectorization_short_regress.q.out       |   18 +-
 .../tez/vectorized_bucketmapjoin1.q.out         |   42 +-
 .../clientpositive/tez/vectorized_context.q.out |   26 +-
 .../vectorized_dynamic_partition_pruning.q.out  |  228 +-
 .../clientpositive/tez/vectorized_mapjoin.q.out |   14 +-
 .../tez/vectorized_nested_mapjoin.q.out         |   66 +-
 .../clientpositive/tez/vectorized_ptf.q.out     |   72 +-
 .../tez/vectorized_shufflejoin.q.out            |   16 +-
 .../udf_case_column_pruning.q.out               |   20 +-
 .../clientpositive/udf_isnull_isnotnull.q.out   |    4 +-
 .../test/results/clientpositive/udf_size.q.out  |    4 +-
 .../clientpositive/udtf_json_tuple.q.out        |   26 +-
 .../clientpositive/udtf_parse_url_tuple.q.out   |   22 +-
 .../test/results/clientpositive/union22.q.out   |    6 +-
 .../test/results/clientpositive/union26.q.out   |   34 +-
 .../test/results/clientpositive/union32.q.out   |   96 +-
 .../test/results/clientpositive/union34.q.out   |   56 +-
 .../clientpositive/unionDistinct_1.q.out        |  214 +-
 .../clientpositive/union_top_level.q.out        |   32 +-
 .../vector_auto_smb_mapjoin_14.q.out            |   72 +-
 .../vector_binary_join_groupby.q.out            |   12 +-
 .../clientpositive/vector_char_mapjoin1.q.out   |   50 +-
 .../clientpositive/vector_decimal_cast.q.out    |    4 +-
 .../vector_decimal_expressions.q.out            |    8 +-
 .../clientpositive/vector_decimal_mapjoin.q.out |   12 +-
 .../clientpositive/vector_decimal_udf.q.out     |   18 +-
 .../results/clientpositive/vector_if_expr.q.out |   10 +-
 .../vector_interval_mapjoin.q.out               |   22 +-
 .../results/clientpositive/vector_join30.q.out  |   64 +-
 .../vector_leftsemi_mapjoin.q.out               | 1356 ++++----
 .../clientpositive/vector_mapjoin_reduce.q.out  |  108 +-
 .../clientpositive/vector_nullsafe_join.q.out   |   48 +-
 .../vector_reduce_groupby_decimal.q.out         |   12 +-
 .../vector_varchar_mapjoin1.q.out               |   50 +-
 .../clientpositive/vectorization_8.q.out        |   16 +-
 .../vectorization_decimal_date.q.out            |    4 +-
 .../clientpositive/vectorization_limit.q.out    |   22 +-
 .../vectorization_offset_limit.q.out            |    8 +-
 .../vectorization_short_regress.q.out           |   18 +-
 .../vectorized_bucketmapjoin1.q.out             |    6 +-
 .../clientpositive/vectorized_context.q.out     |   22 +-
 .../clientpositive/vectorized_mapjoin.q.out     |   12 +-
 .../vectorized_nested_mapjoin.q.out             |   36 +-
 .../results/clientpositive/vectorized_ptf.q.out |   72 +-
 .../clientpositive/vectorized_shufflejoin.q.out |   16 +-
 641 files changed, 22900 insertions(+), 20632 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/hbase-handler/src/test/results/positive/hbase_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index 1ab9877..3907bc9 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -121,7 +121,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
               Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
               Select Operator
                 expressions: key (type: int)
@@ -136,17 +136,17 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
-              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+              predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: UDFToDouble(_col0) (type: double)
                   sort order: +
                   Map-reduce partition columns: UDFToDouble(_col0) (type: double)
-                  Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col0 (type: string), _col1 (type: string)
       Reduce Operator Tree:
         Join Operator
@@ -156,11 +156,11 @@ STAGE PLANS:
             0 UDFToDouble(_col0) (type: double)
             1 UDFToDouble(_col0) (type: double)
           outputColumnNames: _col1, _col2
-          Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col1 (type: string), _col2 (type: string)
             outputColumnNames: _col0, _col1
-            Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
               table:
@@ -175,13 +175,13 @@ STAGE PLANS:
             Reduce Output Operator
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
-              Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
               TopN Hash Memory Usage: 0.1
       Reduce Operator Tree:
         Select Operator
           expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
           Limit
             Number of rows: 20
             Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
@@ -494,19 +494,19 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
-              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+              predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 aggregations: count(key)
                 keys: key (type: string)
                 mode: hash
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                 Reduce Output Operator
                   key expressions: _col0 (type: string)
                   sort order: +
                   Map-reduce partition columns: _col0 (type: string)
-                  Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   value expressions: _col1 (type: bigint)
       Reduce Operator Tree:
         Group By Operator
@@ -514,7 +514,7 @@ STAGE PLANS:
           keys: KEY._col0 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
             table:
@@ -529,7 +529,7 @@ STAGE PLANS:
             alias: hbase_table_1
             Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
             Filter Operator
-              predicate: UDFToDouble(key) is not null (type: boolean)
+              predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
               Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
               Select Operator
                 expressions: key (type: int), value (type: string)
@@ -546,7 +546,7 @@ STAGE PLANS:
               key expressions: UDFToDouble(_col0) (type: double)
               sort order: +
               Map-reduce partition columns: UDFToDouble(_col0) (type: double)
-              Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col1 (type: bigint)
       Reduce Operator Tree:
         Join Operator
@@ -556,14 +556,14 @@ STAGE PLANS:
             0 UDFToDouble(_col0) (type: double)
             1 UDFToDouble(_col0) (type: double)
           outputColumnNames: _col0, _col1, _col3
-          Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: int), _col1 (type: string), UDFToInteger(_col3) (type: int)
             outputColumnNames: _col0, _col1, _col2
-            Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
-              Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
               table:
                   input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
                   output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index e1b60b0..4825a61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -25,6 +25,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.plan.RelOptUtil.InputFinder;
 import org.apache.calcite.plan.RelOptUtil.InputReferencedVisitor;
@@ -32,6 +34,8 @@ import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Join;
 import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
 import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -44,6 +48,7 @@ import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.rex.RexVisitor;
 import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.sql.SqlKind;
@@ -613,6 +618,46 @@ public class HiveCalciteUtil {
     return (new Pair<RelNode, RelNode>(parentOforiginalProjRel, originalProjRel));
   }
 
+  public static boolean isComparisonOp(RexCall call) {
+    return call.getKind().belongsTo(SqlKind.COMPARISON);
+  }
+
+  private static final Function<RexNode, String> REX_STR_FN = new Function<RexNode, String>() {
+                                                              public String apply(RexNode r) {
+                                                                return r.toString();
+                                                              }
+                                                            };
+
+  public static ImmutableList<RexNode> getPredsNotPushedAlready(RelNode inp, List<RexNode> predsToPushDown) {
+    final RelOptPredicateList predicates = RelMetadataQuery.getPulledUpPredicates(inp);
+    final ImmutableSet<String> alreadyPushedPreds = ImmutableSet.copyOf(Lists.transform(
+        predicates.pulledUpPredicates, REX_STR_FN));
+    final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder();
+    for (RexNode r : predsToPushDown) {
+      if (!alreadyPushedPreds.contains(r.toString())) {
+        newConjuncts.add(r);
+      }
+    }
+    return newConjuncts.build();
+  }
+
+  public static RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
+    RexNode typeSafeRex = rex;
+    if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
+      RexBuilder rb = cluster.getRexBuilder();
+      List<RexNode> fixedPredElems = new ArrayList<RexNode>();
+      RelDataType commonType = cluster.getTypeFactory().leastRestrictive(
+          RexUtil.types(((RexCall) rex).getOperands()));
+      for (RexNode rn : ((RexCall) rex).getOperands()) {
+        fixedPredElems.add(rb.ensureType(commonType, rn, true));
+      }
+
+      typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
+    }
+
+    return typeSafeRex;
+  }
+
   public static boolean isDeterministic(RexNode expr) {
     boolean deterministic = true;
 
@@ -635,6 +680,58 @@ public class HiveCalciteUtil {
     return deterministic;
   }
 
+  public static boolean isDeterministicFuncOnLiterals(RexNode expr) {
+    boolean deterministicFuncOnLiterals = true;
+
+    RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) {
+      @Override
+      public Void visitCall(org.apache.calcite.rex.RexCall call) {
+        if (!call.getOperator().isDeterministic()) {
+          throw new Util.FoundOne(call);
+        }
+        return super.visitCall(call);
+      }
+
+      @Override
+      public Void visitInputRef(RexInputRef inputRef) {
+        throw new Util.FoundOne(inputRef);
+      }
+
+      @Override
+      public Void visitLocalRef(RexLocalRef localRef) {
+        throw new Util.FoundOne(localRef);
+      }
+
+      @Override
+      public Void visitOver(RexOver over) {
+        throw new Util.FoundOne(over);
+      }
+
+      @Override
+      public Void visitDynamicParam(RexDynamicParam dynamicParam) {
+        throw new Util.FoundOne(dynamicParam);
+      }
+
+      @Override
+      public Void visitRangeRef(RexRangeRef rangeRef) {
+        throw new Util.FoundOne(rangeRef);
+      }
+
+      @Override
+      public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+        throw new Util.FoundOne(fieldAccess);
+      }
+    };
+
+    try {
+      expr.accept(visitor);
+    } catch (Util.FoundOne e) {
+      deterministicFuncOnLiterals = false;
+    }
+
+    return deterministicFuncOnLiterals;
+  }
+
   public static <T> ImmutableMap<Integer, T> getColInfoMap(List<T> hiveCols,
       int startIndx) {
     Builder<Integer, T> bldr = ImmutableMap.<Integer, T> builder();
@@ -803,6 +900,7 @@ public class HiveCalciteUtil {
 
   public static Set<Integer> getInputRefs(RexNode expr) {
     InputRefsCollector irefColl = new InputRefsCollector(true);
+    expr.accept(irefColl);
     return irefColl.getInputRefSet();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
index 748c86d..c0609d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistinctRowCou
 import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistribution;
 import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdMemory;
 import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdParallelism;
+import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdPredicates;
 import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdRowCount;
 import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSelectivity;
 import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSize;
@@ -75,6 +76,7 @@ public class HiveDefaultRelMetadataProvider {
                     new HiveRelMdParallelism(maxSplitSize).getMetadataProvider(),
                     HiveRelMdDistribution.SOURCE,
                     HiveRelMdCollation.SOURCE,
+                    HiveRelMdPredicates.SOURCE,
                     new DefaultRelMetadataProvider()));
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
index cce3588..02db680 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
@@ -198,7 +198,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
       if (null == partitionList) {
         // we are here either unpartitioned table or partitioned table with no
         // predicates
-        computePartitionList(hiveConf, null);
+        computePartitionList(hiveConf, null, new HashSet<Integer>());
       }
       if (hiveTblMetadata.isPartitioned()) {
         List<Long> rowCounts = StatsUtils.getBasicStatForPartitions(hiveTblMetadata,
@@ -234,8 +234,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
     return sb.toString();
   }
 
-  public void computePartitionList(HiveConf conf, RexNode pruneNode) {
-
+  public void computePartitionList(HiveConf conf, RexNode pruneNode, Set<Integer> partOrVirtualCols) {
     try {
       if (!hiveTblMetadata.isPartitioned() || pruneNode == null
           || InputFinder.bits(pruneNode).length() == 0) {
@@ -248,7 +247,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
 
       // We have valid pruning expressions, only retrieve qualifying partitions
       ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(),
-          HiveCalciteUtil.getInputRefs(pruneNode), this.getRelOptSchema().getTypeFactory()));
+          partOrVirtualCols, this.getRelOptSchema().getTypeFactory()));
 
       partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(),
           partitionCache);
@@ -287,7 +286,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
     if (null == partitionList) {
       // We could be here either because its an unpartitioned table or because
       // there are no pruning predicates on a partitioned table.
-      computePartitionList(hiveConf, null);
+      computePartitionList(hiveConf, null, new HashSet<Integer>());
     }
 
     // 2. Obtain Col Stats for Non Partition Cols

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
index be28828..024ba46 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
@@ -34,4 +34,8 @@ public class TraitsUtil {
   public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) {
     return cluster.traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY);
   }
+
+  public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster, RelTraitSet traitsFromInput) {
+    return RelTraitSet.createEmpty().merge(traitsFromInput).merge(getDefaultTraitSet(cluster));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
index 4b7887a..db625c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
@@ -50,6 +50,7 @@ public class HiveProject extends Project implements HiveRelNode {
   public static final ProjectFactory DEFAULT_PROJECT_FACTORY = new HiveProjectFactoryImpl();
 
   private final List<Integer>        virtualCols;
+  private boolean isSysnthetic;
 
   /**
    * Creates a HiveProject.
@@ -165,10 +166,14 @@ public class HiveProject extends Project implements HiveRelNode {
   }
 
   @Override
-  public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps,
-      RelDataType rowType) {
+  public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps, RelDataType rowType) {
     assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
-    return new HiveProject(getCluster(), traitSet, input, exps, rowType, getFlags());
+    HiveProject hp = new HiveProject(getCluster(), traitSet, input, exps, rowType, getFlags());
+    if (this.isSysnthetic()) {
+      hp.setSynthetic();
+    }
+
+    return hp;
   }
 
   @Override
@@ -184,6 +189,16 @@ public class HiveProject extends Project implements HiveRelNode {
     return virtualCols;
   }
 
+  // TODO: this should come through RelBuilder to the constructor as opposed to
+  // set method. This requires calcite change
+  public void setSynthetic() {
+    this.isSysnthetic = true;
+  }
+
+  public boolean isSysnthetic() {
+    return isSysnthetic;
+  }
+
   /**
    * Implementation of {@link ProjectFactory} that returns
    * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject}
@@ -196,9 +211,9 @@ public class HiveProject extends Project implements HiveRelNode {
         List<? extends RexNode> childExprs, List<String> fieldNames) {
       RelOptCluster cluster = child.getCluster();
       RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames);
+      RelTraitSet trait = TraitsUtil.getDefaultTraitSet(cluster, child.getTraitSet());
       RelNode project = HiveProject.create(cluster, child,
-          childExprs, rowType,
-          child.getTraitSet(), Collections.<RelCollation> emptyList());
+          childExprs, rowType, trait, Collections.<RelCollation> emptyList());
 
       return project;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
index 446dc73..5788805 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
@@ -37,6 +37,7 @@ import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Pair;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
 import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.ColStatistics;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableSet;
 
 
 /**
@@ -61,6 +63,7 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
   private final String tblAlias;
   private final String concatQbIDAlias;
   private final boolean useQBIdInDigest;
+  private final ImmutableSet<Integer> viurtualOrPartColIndxsInTS;
 
   public String getTableAlias() {
     return tblAlias;
@@ -94,7 +97,9 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
     this.tblAlias = alias;
     this.concatQbIDAlias = concatQbIDAlias;
     this.hiveTableScanRowType = newRowtype;
-    this.neededColIndxsFrmReloptHT = buildNeededColIndxsFrmReloptHT(table.getRowType(), newRowtype);
+    Pair<ImmutableList<Integer>, ImmutableSet<Integer>> colIndxPair = buildColIndxsFrmReloptHT(table, newRowtype);
+    this.neededColIndxsFrmReloptHT = colIndxPair.getKey();
+    this.viurtualOrPartColIndxsInTS = colIndxPair.getValue();
     this.useQBIdInDigest = useQBIdInDigest;
   }
 
@@ -183,7 +188,13 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
         fieldNames));
 
     // 5. Add Proj on top of TS
-    return relBuilder.push(newHT).project(exprList, new ArrayList<String>(fieldNames)).build();
+    HiveProject hp = (HiveProject) relBuilder.push(newHT)
+        .project(exprList, new ArrayList<String>(fieldNames)).build();
+
+    // 6. Set synthetic flag, so that we would push filter below this one
+    hp.setSynthetic();
+
+    return hp;
   }
 
   public List<Integer> getNeededColIndxsFrmReloptHT() {
@@ -194,17 +205,36 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
     return hiveTableScanRowType;
   }
 
-  private static ImmutableList<Integer> buildNeededColIndxsFrmReloptHT(RelDataType htRowtype,
-      RelDataType scanRowType) {
+  public Set<Integer> getPartOrVirtualCols() {
+    return viurtualOrPartColIndxsInTS;
+  }
+
+  private static Pair<ImmutableList<Integer>, ImmutableSet<Integer>> buildColIndxsFrmReloptHT(
+      RelOptHiveTable relOptHTable, RelDataType scanRowType) {
+    RelDataType relOptHtRowtype = relOptHTable.getRowType();
+    ImmutableList<Integer> neededColIndxsFrmReloptHT;
     Builder<Integer> neededColIndxsFrmReloptHTBldr = new ImmutableList.Builder<Integer>();
-    Map<String, Integer> colNameToPosInReloptHT = HiveCalciteUtil.getRowColNameIndxMap(htRowtype
-        .getFieldList());
+    ImmutableSet<Integer> viurtualOrPartColIndxsInTS;
+    ImmutableSet.Builder<Integer> viurtualOrPartColIndxsInTSBldr = new ImmutableSet.Builder<Integer>();
+
+    Map<String, Integer> colNameToPosInReloptHT = HiveCalciteUtil
+        .getRowColNameIndxMap(relOptHtRowtype.getFieldList());
     List<String> colNamesInScanRowType = scanRowType.getFieldNames();
 
+    int partOrVirtualColStartPosInrelOptHtRowtype = relOptHTable.getNonPartColumns().size();
+    int tmp;
     for (int i = 0; i < colNamesInScanRowType.size(); i++) {
-      neededColIndxsFrmReloptHTBldr.add(colNameToPosInReloptHT.get(colNamesInScanRowType.get(i)));
+      tmp = colNameToPosInReloptHT.get(colNamesInScanRowType.get(i));
+      neededColIndxsFrmReloptHTBldr.add(tmp);
+      if (tmp >= partOrVirtualColStartPosInrelOptHtRowtype) {
+        viurtualOrPartColIndxsInTSBldr.add(i);
+      }
     }
 
-    return neededColIndxsFrmReloptHTBldr.build();
+    neededColIndxsFrmReloptHT = neededColIndxsFrmReloptHTBldr.build();
+    viurtualOrPartColIndxsInTS = viurtualOrPartColIndxsInTSBldr.build();
+
+    return new Pair<ImmutableList<Integer>, ImmutableSet<Integer>>(neededColIndxsFrmReloptHT,
+        viurtualOrPartColIndxsInTS);
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
new file mode 100644
index 0000000..0e5c731
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.RelFactories.FilterFactory;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rex.RexNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+
+public class HiveFilterAggregateTransposeRule extends FilterAggregateTransposeRule {
+
+  public HiveFilterAggregateTransposeRule(Class<? extends Filter> filterClass,
+      FilterFactory filterFactory, Class<? extends Aggregate> aggregateClass) {
+    super(filterClass, filterFactory, aggregateClass);
+  }
+
+  @Override
+  public boolean matches(RelOptRuleCall call) {
+    final Filter filterRel = call.rel(0);
+    RexNode condition = filterRel.getCondition();
+    if (!HiveCalciteUtil.isDeterministic(condition)) {
+      return false;
+    }
+
+    return super.matches(call);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
new file mode 100644
index 0000000..8321504
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.core.RelFactories.FilterFactory;
+import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+
+//TODO: Remove this once Calcite FilterProjectTransposeRule can take rule operand
+public class HiveFilterProjectTSTransposeRule extends RelOptRule {
+
+  private final RelFactories.FilterFactory  filterFactory;
+  private final RelFactories.ProjectFactory projectFactory;
+
+  public HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
+      FilterFactory filterFactory, Class<? extends Project> projectClass,
+      ProjectFactory projectFactory, Class<? extends TableScan> tsClass) {
+    super(operand(filterClass, operand(projectClass, operand(tsClass, none()))));
+    this.filterFactory = filterFactory;
+    this.projectFactory = projectFactory;
+  }
+
+  @Override
+  public boolean matches(RelOptRuleCall call) {
+    final HiveProject projRel = call.rel(1);
+
+    // Assumption:
+    // 1. This will be run last after PP, Col Pruning in the PreJoinOrder
+    // optimizations.
+    // 2. If ProjectRel is not synthetic then PPD would have already pushed
+    // relevant pieces down and hence no point in running PPD again.
+    // 3. For synthetic Projects we don't care about non deterministic UDFs
+    if (!projRel.isSysnthetic()) {
+      return false;
+    }
+
+    return super.matches(call);
+  }
+
+  // ~ Methods ----------------------------------------------------------------
+
+  // implement RelOptRule
+  public void onMatch(RelOptRuleCall call) {
+    final Filter filter = call.rel(0);
+    final Project project = call.rel(1);
+
+    if (RexOver.containsOver(project.getProjects(), null)) {
+      // In general a filter cannot be pushed below a windowing calculation.
+      // Applying the filter before the aggregation function changes
+      // the results of the windowing invocation.
+      //
+      // When the filter is on the PARTITION BY expression of the OVER clause
+      // it can be pushed down. For now we don't support this.
+      return;
+    }
+
+    if (RexUtil.containsCorrelation(filter.getCondition())) {
+      // If there is a correlation condition anywhere in the filter, don't
+      // push this filter past project since in some cases it can prevent a
+      // Correlate from being de-correlated.
+      return;
+    }
+
+    // convert the filter to one that references the child of the project
+    RexNode newCondition = RelOptUtil.pushPastProject(filter.getCondition(), project);
+
+    // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
+    // nullable and not-nullable conditions, but a CAST might get in the way of
+    // other rewrites.
+    final RelDataTypeFactory typeFactory = filter.getCluster().getTypeFactory();
+    if (RexUtil.isNullabilityCast(typeFactory, newCondition)) {
+      newCondition = ((RexCall) newCondition).getOperands().get(0);
+    }
+
+    RelNode newFilterRel = filterFactory == null ? filter.copy(filter.getTraitSet(),
+        project.getInput(), newCondition) : filterFactory.createFilter(project.getInput(),
+        newCondition);
+
+    RelNode newProjRel = projectFactory == null ? project.copy(project.getTraitSet(), newFilterRel,
+        project.getProjects(), project.getRowType()) : projectFactory.createProject(newFilterRel,
+        project.getProjects(), project.getRowType().getFieldNames());
+
+    call.transformTo(newProjRel);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
index 29deed9..703c8c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptPredicateList;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptRuleCall;
@@ -26,6 +27,7 @@ import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Join;
 import org.apache.calcite.rel.core.RelFactories;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -35,6 +37,7 @@ import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.util.Util;
 import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
 import org.apache.hive.common.util.AnnotationUtils;
 
@@ -68,14 +71,21 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule {
 
   @Override public void onMatch(RelOptRuleCall call) {
     Join join = call.rel(0);
+
+    // Register that we have visited this operator in this rule
+    HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
+    if (registry != null) {
+      registry.registerVisited(this, join);
+    }
+
     RelOptPredicateList preds = RelMetadataQuery.getPulledUpPredicates(join);
 
     RexBuilder rB = join.getCluster().getRexBuilder();
     RelNode lChild = call.rel(1);
     RelNode rChild = call.rel(2);
 
-    List<RexNode> leftPreds = getValidPreds(preds.leftInferredPredicates, lChild.getRowType().getFieldList());
-    List<RexNode> rightPreds = getValidPreds(preds.rightInferredPredicates, rChild.getRowType().getFieldList());
+    List<RexNode> leftPreds = getValidPreds(join.getCluster(), lChild, preds.leftInferredPredicates, lChild.getRowType());
+    List<RexNode> rightPreds = getValidPreds(join.getCluster(), rChild, preds.rightInferredPredicates, rChild.getRowType());
 
     if (leftPreds.isEmpty() && rightPreds.isEmpty()) {
       return;
@@ -97,11 +107,17 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule {
         lChild, rChild, join.getJoinType(), join.isSemiJoinDone());
     call.getPlanner().onCopy(join, newRel);
 
+    // We register new Join rel so we do not fire the rule on them again
+    if (registry != null) {
+      registry.registerVisited(this, newRel);
+    }
+
     call.transformTo(newRel);
   }
 
-  private ImmutableList<RexNode> getValidPreds (List<RexNode> rexs, List<RelDataTypeField> types) {
-    InputRefValidator validator = new InputRefValidator(types);
+  private ImmutableList<RexNode> getValidPreds(RelOptCluster cluster, RelNode rn,
+      List<RexNode> rexs, RelDataType rType) {
+    InputRefValidator validator = new InputRefValidator(rType.getFieldList());
     List<RexNode> valids = new ArrayList<RexNode>(rexs.size());
     for (RexNode rex : rexs) {
       try {
@@ -111,7 +127,25 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule {
         Util.swallow(e, null);
       }
     }
-    return ImmutableList.copyOf(valids);
+
+    return HiveCalciteUtil.getPredsNotPushedAlready(rn, valids);
+  }
+
+  private RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
+    RexNode typeSafeRex = rex;
+    if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
+      RexBuilder rb = cluster.getRexBuilder();
+      List<RexNode> fixedPredElems = new ArrayList<RexNode>();
+      RelDataType commonType = cluster.getTypeFactory().leastRestrictive(
+          RexUtil.types(((RexCall) rex).getOperands()));
+      for (RexNode rn : ((RexCall) rex).getOperands()) {
+        fixedPredElems.add(rb.ensureType(commonType, rn, true));
+      }
+
+      typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
+    }
+
+    return typeSafeRex;
   }
 
   private static class InputRefValidator  extends RexVisitorImpl<Void> {

http://git-wip-us.apache.org/repos/asf/hive/blob/08f73adc/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
index ba28055..d8c3a22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
@@ -52,6 +52,6 @@ public class HivePartitionPruneRule extends RelOptRule {
     Pair<RexNode, RexNode> predicates = PartitionPrune
         .extractPartitionPredicates(filter.getCluster(), hiveTable, predicate);
     RexNode partColExpr = predicates.left;
-    hiveTable.computePartitionList(conf, partColExpr);
+    hiveTable.computePartitionList(conf, partColExpr, tScan.getPartOrVirtualCols());
   }
 }


Mime
View raw message