hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1613661 [1/9] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src/test/results/clientpositive/ hbase-handler/src/test/results/positive/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/...
Date Sat, 26 Jul 2014 15:39:59 GMT
Author: hashutosh
Date: Sat Jul 26 15:39:55 2014
New Revision: 1613661

URL: http://svn.apache.org/r1613661
Log:
HIVE-5771 : Constant propagation optimizer for Hive (Ted Xu via Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
    hive/trunk/ql/src/test/queries/clientpositive/constprog1.q
    hive/trunk/ql/src/test/queries/clientpositive/constprog2.q
    hive/trunk/ql/src/test/queries/clientpositive/constprog_dp.q
    hive/trunk/ql/src/test/queries/clientpositive/constprog_type.q
    hive/trunk/ql/src/test/results/clientpositive/constprog1.q.out
    hive/trunk/ql/src/test/results/clientpositive/constprog2.q.out
    hive/trunk/ql/src/test/results/clientpositive/constprog_dp.q.out
    hive/trunk/ql/src/test/results/clientpositive/constprog_type.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml.template
    hive/trunk/contrib/src/test/results/clientpositive/udf_example_add.q.out
    hive/trunk/contrib/src/test/results/clientpositive/udf_example_format.q.out
    hive/trunk/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
    hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
    hive/trunk/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedMergeJoinProc.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/UDFCurrentDB.java
    hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_18.q
    hive/trunk/ql/src/test/queries/clientpositive/subquery_views.q
    hive/trunk/ql/src/test/queries/clientpositive/union27.q
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_part.q.out
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join14_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/cast1.q.out
    hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
    hive/trunk/ql/src/test/results/clientpositive/column_access_stats.q.out
    hive/trunk/ql/src/test/results/clientpositive/combine2_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/create_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/cross_product_check_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/cross_product_check_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/cte_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/explain_logical.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_ppd.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_empty.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_update.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_stale.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_stale_partitioned.q.out
    hive/trunk/ql/src/test/results/clientpositive/infer_const_type.q.out
    hive/trunk/ql/src/test/results/clientpositive/input18.q.out
    hive/trunk/ql/src/test/results/clientpositive/input23.q.out
    hive/trunk/ql/src/test/results/clientpositive/input26.q.out
    hive/trunk/ql/src/test/results/clientpositive/input38.q.out
    hive/trunk/ql/src/test/results/clientpositive/input39_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/input6.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part2.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part4.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part6.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert1.q.out
    hive/trunk/ql/src/test/results/clientpositive/join14_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/join38.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_14.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_3.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_6.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_7.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_query_multiskew_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_query_multiskew_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_query_oneskew_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/list_bucket_query_oneskew_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/literal_double.q.out
    hive/trunk/ql/src/test/results/clientpositive/macro.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapjoin1.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
    hive/trunk/ql/src/test/results/clientpositive/nullgroup4.q.out
    hive/trunk/ql/src/test/results/clientpositive/nullgroup4_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/num_op_type_conv.q.out
    hive/trunk/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out
    hive/trunk/ql/src/test/results/clientpositive/pcr.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_constant_where.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_repeated_alias.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_udf_col.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_union_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/query_result_fileformat.q.out
    hive/trunk/ql/src/test/results/clientpositive/quotedid_basic.q.out
    hive/trunk/ql/src/test/results/clientpositive/quotedid_partition.q.out
    hive/trunk/ql/src/test/results/clientpositive/regex_col.q.out
    hive/trunk/ql/src/test/results/clientpositive/regexp_extract.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
    hive/trunk/ql/src/test/results/clientpositive/select_dummy_source.q.out
    hive/trunk/ql/src/test/results/clientpositive/select_unquote_and.q.out
    hive/trunk/ql/src/test/results/clientpositive/select_unquote_not.q.out
    hive/trunk/ql/src/test/results/clientpositive/select_unquote_or.q.out
    hive/trunk/ql/src/test/results/clientpositive/set_processor_namespaces.q.out
    hive/trunk/ql/src/test/results/clientpositive/set_variable_sub.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_18.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
    hive/trunk/ql/src/test/results/clientpositive/stats_empty_dyn_part.q.out
    hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_exists.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_exists_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_in.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_in_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notexists.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notin.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_views.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/insert1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/subquery_exists.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/subquery_in.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/transform_ppr1.q.out
    hive/trunk/ql/src/test/results/clientpositive/transform_ppr1.q.out
    hive/trunk/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out
    hive/trunk/ql/src/test/results/clientpositive/type_cast_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/type_widening.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf1.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf4.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf5.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf6.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf7.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf9.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_10_trims.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_E.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_PI.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_abs.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_ascii.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_between.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_case.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_coalesce.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_current_database.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_degrees.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_elt.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_format_number.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_hash.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_hour.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_if.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_instr.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_isnull_isnotnull.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_like.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_locate.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_lower.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_lpad.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_minute.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_nvl.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_parse_url.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_printf.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_radians.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_reflect2.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_repeat.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_rpad.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_second.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_sign.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_space.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_when.q.out
    hive/trunk/ql/src/test/results/clientpositive/union20.q.out
    hive/trunk/ql/src/test/results/clientpositive/union27.q.out
    hive/trunk/ql/src/test/results/clientpositive/union33.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_remove_19.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
    hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat Jul 26 15:39:55 2014
@@ -983,6 +983,8 @@ public class HiveConf extends Configurat
         "Whether to transitively replicate predicate filters over equijoin conditions."),
     HIVEPPDREMOVEDUPLICATEFILTERS("hive.ppd.remove.duplicatefilters", true,
         "Whether to push predicates down into storage handlers.  Ignored when hive.optimize.ppd is false."),
+    // Constant propagation optimizer
+    HIVEOPTCONSTANTPROPAGATION("hive.optimize.constant.propagation", true, "Whether to enable constant propagation optimizer"),
     HIVEMETADATAONLYQUERIES("hive.optimize.metadataonly", true, ""),
     HIVENULLSCANOPTIMIZE("hive.optimize.null.scan", true, "Dont scan relations which are guaranteed to not generate any rows"),
     HIVEOPTPPD_STORAGE("hive.optimize.ppd.storage", true,
@@ -2250,7 +2252,6 @@ public class HiveConf extends Configurat
     return hiveServer2SiteUrl;
   }
 
-
   /**
    * @return the user name set in hadoop.job.ugi param or the current user from System
    * @throws IOException

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Sat Jul 26 15:39:55 2014
@@ -1,5 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with
    this work for additional information regarding copyright ownership.
@@ -14,7 +16,8 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
---><configuration>
+-->
+<configuration>
   <!-- WARNING!!! This file is auto generated for documentation purposes ONLY! -->
   <!-- WARNING!!! Any changes you make to this file will be ignored by Hive.   -->
   <!-- WARNING!!! You must make your changes in hive-site.xml instead.         -->
@@ -1703,6 +1706,11 @@
     <description>Whether to push predicates down into storage handlers.  Ignored when hive.optimize.ppd is false.</description>
   </property>
   <property>
+    <key>hive.optimize.constant.propagation</key>
+    <value>true</value>
+    <description>Whether to enable constant propagation optimizer</description>
+  </property>
+  <property>
     <key>hive.optimize.metadataonly</key>
     <value>true</value>
     <description/>

Modified: hive/trunk/contrib/src/test/results/clientpositive/udf_example_add.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/udf_example_add.q.out?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/udf_example_add.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/udf_example_add.q.out Sat Jul 26 15:39:55 2014
@@ -36,7 +36,7 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE
             Select Operator
-              expressions: example_add(1, 2) (type: int), example_add(1, 2, 3) (type: int), example_add(1, 2, 3, 4) (type: int), example_add(1.1, 2.2) (type: double), example_add(1.1, 2.2, 3.3) (type: double), example_add(1.1, 2.2, 3.3, 4.4) (type: double), example_add(1, 2, 3, 4.4) (type: double)
+              expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003 (type: double), 6.6 (type: double), 11.0 (type: double), 10.4 (type: double)
               outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
               Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE
               Limit

Modified: hive/trunk/contrib/src/test/results/clientpositive/udf_example_format.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/udf_example_format.q.out?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/udf_example_format.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/udf_example_format.q.out Sat Jul 26 15:39:55 2014
@@ -30,7 +30,7 @@ STAGE PLANS:
             alias: src
             Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE
             Select Operator
-              expressions: example_format('abc') (type: string), example_format('%1$s', 1.1) (type: string), example_format('%1$s %2$e', 1.1, 1.2) (type: string), example_format('%1$x %2$o %3$d', 10, 10, 10) (type: string)
+              expressions: 'abc' (type: string), '1.1' (type: string), '1.1 1.200000e+00' (type: string), 'a 12 10' (type: string)
               outputColumnNames: _col0, _col1, _col2, _col3
               Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE
               Limit

Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_ppd_key_range.q.out Sat Jul 26 15:39:55 2014
@@ -193,10 +193,10 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: hbase_pushdown
-            filterExpr: (key >= UDFToString((40 + 50))) (type: boolean)
+            filterExpr: (key >= '90') (type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: (key >= UDFToString((40 + 50))) (type: boolean)
+              predicate: (key >= '90') (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string)

Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_pushdown.q.out Sat Jul 26 15:39:55 2014
@@ -43,7 +43,7 @@ STAGE PLANS:
               predicate: (key = 90) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
-                expressions: key (type: int), value (type: string)
+                expressions: 90 (type: int), value (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 File Output Operator
@@ -235,7 +235,7 @@ STAGE PLANS:
               predicate: (((key = 80) and (key = 90)) and (value like '%90%')) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
-                expressions: key (type: int), value (type: string)
+                expressions: 90 (type: int), value (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 File Output Operator
@@ -398,7 +398,7 @@ STAGE PLANS:
               predicate: (key = 90) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
-                expressions: key (type: int), value (type: string)
+                expressions: 90 (type: int), value (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 File Output Operator

Modified: hive/trunk/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/ppd_key_ranges.q.out Sat Jul 26 15:39:55 2014
@@ -191,7 +191,7 @@ STAGE PLANS:
               predicate: (((key >= 9) and (key < 17)) and (key = 11)) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
-                expressions: key (type: int), value (type: string)
+                expressions: 11 (type: int), value (type: string)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 File Output Operator

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java Sat Jul 26 15:39:55 2014
@@ -220,4 +220,8 @@ public class ColumnInfo implements Seria
 
     return true;
   }
+
+  public void setObjectinspector(ObjectInspector writableObjectInspector) {
+    this.objectInspector = writableObjectInspector;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java Sat Jul 26 15:39:55 2014
@@ -104,4 +104,9 @@ public abstract class ExprNodeEvaluator<
   public ExprNodeEvaluator[] getChildren() {
     return null;
   }
+  
+  @Override
+  public String toString() {
+    return "ExprNodeEvaluator[" + expr + "]";
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Sat Jul 26 15:39:55 2014
@@ -131,7 +131,7 @@ public class MapJoinOperator extends Abs
     int bigPos = conf.getPosBigTable();
     List<ObjectInspector> valueOI = new ArrayList<ObjectInspector>();
     for (int i = 0; i < valueIndex.length; i++) {
-      if (valueIndex[i] >= 0) {
+      if (valueIndex[i] >= 0 && !joinKeysObjectInspectors[bigPos].isEmpty()) {
         valueOI.add(joinKeysObjectInspectors[bigPos].get(valueIndex[i]));
       } else {
         valueOI.add(inspectors.get(i));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Sat Jul 26 15:39:55 2014
@@ -272,6 +272,7 @@ public class ReduceSinkOperator extends 
         // TODO: this is fishy - we init object inspectors based on first tag. We
         //       should either init for each tag, or if rowInspector doesn't really
         //       matter, then we can create this in ctor and get rid of firstRow.
+        LOG.info("keys are " + conf.getOutputKeyColumnNames() + " num distributions: " + conf.getNumDistributionKeys());
         keyObjectInspector = initEvaluatorsAndReturnStruct(keyEval,
             distinctColIndices,
             conf.getOutputKeyColumnNames(), numDistributionKeys, rowInspector);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java Sat Jul 26 15:39:55 2014
@@ -32,8 +32,7 @@ import org.apache.hadoop.hive.serde2.obj
 /**
  * Select operator implementation.
  */
-public class SelectOperator extends Operator<SelectDesc> implements
-    Serializable {
+public class SelectOperator extends Operator<SelectDesc> implements Serializable {
 
   private static final long serialVersionUID = 1L;
   protected transient ExprNodeEvaluator[] eval;
@@ -60,10 +59,9 @@ public class SelectOperator extends Oper
       }
     }
     output = new Object[eval.length];
-    LOG.info("SELECT "
-        + ((StructObjectInspector) inputObjInspectors[0]).getTypeName());
-    outputObjInspector = initEvaluatorsAndReturnStruct(eval, conf
-        .getOutputColumnNames(), inputObjInspectors[0]);
+    LOG.info("SELECT " + ((StructObjectInspector) inputObjInspectors[0]).getTypeName());
+    outputObjInspector = initEvaluatorsAndReturnStruct(eval, conf.getOutputColumnNames(),
+        inputObjInspectors[0]);
     initializeChildren(hconf);
   }
 
@@ -81,8 +79,7 @@ public class SelectOperator extends Oper
     } catch (HiveException e) {
       throw e;
     } catch (RuntimeException e) {
-      throw new HiveException("Error evaluating "
-          + conf.getColList().get(i).getExprString(), e);
+      throw new HiveException("Error evaluating " + conf.getColList().get(i).getExprString(), e);
     }
     forward(output, outputObjInspector);
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1613661&r1=1613660&r2=1613661&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Sat Jul 26 15:39:55 2014
@@ -30,6 +30,7 @@ import java.util.Stack;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
@@ -87,6 +88,7 @@ public final class ColumnPrunerProcFacto
    * Node Processor for Column Pruning on Filter Operators.
    */
   public static class ColumnPrunerFilterProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       FilterOperator op = (FilterOperator) nd;
@@ -120,6 +122,7 @@ public final class ColumnPrunerProcFacto
    * Node Processor for Column Pruning on Group By Operators.
    */
   public static class ColumnPrunerGroupByProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       GroupByOperator op = (GroupByOperator) nd;
@@ -154,6 +157,7 @@ public final class ColumnPrunerProcFacto
   }
 
   public static class ColumnPrunerScriptProc implements NodeProcessor {
+    @Override
     @SuppressWarnings("unchecked")
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
@@ -224,6 +228,7 @@ public final class ColumnPrunerProcFacto
    *   and update the RR & signature on the PTFOp.
    */
   public static class ColumnPrunerPTFProc extends ColumnPrunerScriptProc {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
 
@@ -327,6 +332,7 @@ public final class ColumnPrunerProcFacto
    * The Default Node Processor for Column Pruning.
    */
   public static class ColumnPrunerDefaultProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
@@ -351,6 +357,7 @@ public final class ColumnPrunerProcFacto
    * store needed columns in tableScanDesc.
    */
   public static class ColumnPrunerTableScanProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       TableScanOperator scanOp = (TableScanOperator) nd;
@@ -426,6 +433,7 @@ public final class ColumnPrunerProcFacto
    * The Node Processor for Column Pruning on Reduce Sink Operators.
    */
   public static class ColumnPrunerReduceSinkProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       ReduceSinkOperator op = (ReduceSinkOperator) nd;
@@ -435,6 +443,7 @@ public final class ColumnPrunerProcFacto
 
       List<String> colLists = new ArrayList<String>();
       ArrayList<ExprNodeDesc> keys = conf.getKeyCols();
+      LOG.debug("Reduce Sink Operator " + op.getIdentifier() + " key:" + keys);
       for (ExprNodeDesc key : keys) {
         colLists = Utilities.mergeUniqElems(colLists, key.getCols());
       }
@@ -456,7 +465,6 @@ public final class ColumnPrunerProcFacto
 
       if (childCols != null) {
         boolean[] flags = new boolean[valCols.size()];
-        Map<String, ExprNodeDesc> exprMap = op.getColumnExprMap();
 
         for (String childCol : childCols) {
           int index = valColNames.indexOf(Utilities.removeValueTag(childCol));
@@ -497,6 +505,7 @@ public final class ColumnPrunerProcFacto
    * The Node Processor for Column Pruning on Lateral View Join Operators.
    */
   public static class ColumnPrunerLateralViewJoinProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       LateralViewJoinOperator op = (LateralViewJoinOperator) nd;
@@ -585,6 +594,7 @@ public final class ColumnPrunerProcFacto
    * The Node Processor for Column Pruning on Select Operators.
    */
   public static class ColumnPrunerSelectProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       SelectOperator op = (SelectOperator) nd;
@@ -748,6 +758,12 @@ public final class ColumnPrunerProcFacto
           nm = oldRR.reverseLookup(outputCol);
         }
 
+        // In case there are multiple columns referenced to the same column name, we won't
+        // do row resolve once more because the ColumnInfo in row resolver is already removed
+        if (nm == null) {
+          continue;
+        }
+
         // Only remove information of a column if it is not a key,
         // i.e. this column is not appearing in keyExprs of the RS
         if (ExprNodeDescUtils.indexOf(outputColExpr, keyExprs) == -1) {
@@ -795,6 +811,7 @@ public final class ColumnPrunerProcFacto
    * The Node Processor for Column Pruning on Join Operators.
    */
   public static class ColumnPrunerJoinProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
       JoinOperator op = (JoinOperator) nd;
@@ -817,9 +834,10 @@ public final class ColumnPrunerProcFacto
    * The Node Processor for Column Pruning on Map Join Operators.
    */
   public static class ColumnPrunerMapJoinProc implements NodeProcessor {
+    @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
         Object... nodeOutputs) throws SemanticException {
-      MapJoinOperator op = (MapJoinOperator) nd;
+      AbstractMapJoinOperator<MapJoinDesc> op = (AbstractMapJoinOperator<MapJoinDesc>) nd;
       pruneJoinOperator(ctx, op, op.getConf(), op.getColumnExprMap(), op
           .getConf().getRetainList(), true);
       return null;
@@ -878,6 +896,7 @@ public final class ColumnPrunerProcFacto
     List<Operator<? extends OperatorDesc>> childOperators = op
         .getChildOperators();
 
+    LOG.info("JOIN " + op.getIdentifier() + " oldExprs: " + conf.getExprs());
     List<String> childColLists = cppCtx.genColLists(op);
     if (childColLists == null) {
         return;
@@ -985,6 +1004,7 @@ public final class ColumnPrunerProcFacto
       rs.add(col);
     }
 
+    LOG.info("JOIN " + op.getIdentifier() + " newExprs: " + conf.getExprs());
     op.setColumnExprMap(newColExprMap);
     conf.setOutputColumnNames(outputCols);
     op.getSchema().setSignature(rs);

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java?rev=1613661&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagate.java Sat Jul 26 15:39:55 2014
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.FilterOperator;
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.exec.ScriptOperator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.parse.OpParseContext;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+
+/**
+ * Implementation of one of the rule-based optimization steps. ConstantPropagate traverse the DAG
+ * from root to child. For each conditional expression, process as follows:
+ *
+ * 1. Fold constant expression: if the expression is a UDF and all parameters are constant.
+ *
+ * 2. Shortcut expression: if the expression is a logical operator and it can be shortcut by
+ * some constants of its parameters.
+ *
+ * 3. Propagate expression: if the expression is an assignment like column=constant, the expression
+ * will be propagate to parents to see if further folding operation is possible.
+ */
+public class ConstantPropagate implements Transform {
+
+  private static final Log LOG = LogFactory.getLog(ConstantPropagate.class);
+  protected ParseContext pGraphContext;
+  private Map<Operator<? extends OperatorDesc>, OpParseContext> opToParseCtxMap;
+
+  public ConstantPropagate() {}
+
+  /**
+   * Transform the query tree.
+   *
+   * @param pactx
+   *        the current parse context
+   */
+  @Override
+  public ParseContext transform(ParseContext pactx) throws SemanticException {
+    if (pactx.getConf().getBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED)) {
+      // Constant propagate is currently conflict with vectorizer, disabling constant propagate
+      //    if the later is enabled.
+      return pactx;
+    }
+    if (pactx.getConf().getBoolVar(ConfVars.HIVEOPTSORTMERGEBUCKETMAPJOIN)) {
+      return pactx;
+    }
+    pGraphContext = pactx;
+    opToParseCtxMap = pGraphContext.getOpParseCtx();
+
+    // generate pruned column list for all relevant operators
+    ConstantPropagateProcCtx cppCtx = new ConstantPropagateProcCtx(opToParseCtxMap);
+
+    // create a walker which walks the tree in a DFS manner while maintaining
+    // the operator stack. The dispatcher
+    // generates the plan from the operator tree
+    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+
+    opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getFilterProc());
+    opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getGroupByProc());
+    opRules.put(new RuleRegExp("R3", SelectOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getSelectProc());
+    opRules.put(new RuleRegExp("R4", FileSinkOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getFileSinkProc());
+    opRules.put(new RuleRegExp("R5", ReduceSinkOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getReduceSinkProc());
+    opRules.put(new RuleRegExp("R6", JoinOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getJoinProc());
+    opRules.put(new RuleRegExp("R7", TableScanOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getTableScanProc());
+    opRules.put(new RuleRegExp("R8", ScriptOperator.getOperatorName() + "%"),
+        ConstantPropagateProcFactory.getStopProc());
+
+    // The dispatcher fires the processor corresponding to the closest matching
+    // rule and passes the context along
+    Dispatcher disp = new DefaultRuleDispatcher(ConstantPropagateProcFactory
+        .getDefaultProc(), opRules, cppCtx);
+    GraphWalker ogw = new ConstantPropagateWalker(disp);
+
+    // Create a list of operator nodes to start the walking.
+    ArrayList<Node> topNodes = new ArrayList<Node>();
+    topNodes.addAll(pGraphContext.getTopOps().values());
+    ogw.startWalking(topNodes, null);
+    for (Operator<? extends Serializable> opToDelete : cppCtx.getOpToDelete()) {
+      if (opToDelete.getParentOperators() == null || opToDelete.getParentOperators().size() != 1) {
+        throw new RuntimeException("Error pruning operator " + opToDelete
+            + ". It should have only 1 parent.");
+      }
+      opToDelete.getParentOperators().get(0).removeChildAndAdoptItsChildren(opToDelete);
+    }
+    return pGraphContext;
+  }
+
+
+  /**
+   * Walks the op tree in root first order.
+   */
+  public static class ConstantPropagateWalker extends DefaultGraphWalker {
+
+    public ConstantPropagateWalker(Dispatcher disp) {
+      super(disp);
+    }
+
+    @Override
+    public void walk(Node nd) throws SemanticException {
+
+      List<Node> parents = ((Operator) nd).getParentOperators();
+      if ((parents == null)
+          || getDispatchedList().containsAll(parents)) {
+        opStack.push(nd);
+
+        // all children are done or no need to walk the children
+        dispatch(nd, opStack);
+        opStack.pop();
+      } else {
+        getToWalk().removeAll(parents);
+        getToWalk().add(0, nd);
+        getToWalk().addAll(0, parents);
+        return;
+      }
+
+      // move all the children to the front of queue
+      List<? extends Node> children = nd.getChildren();
+      if (children != null) {
+        getToWalk().removeAll(children);
+        getToWalk().addAll(children);
+      }
+    }
+  }
+
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java?rev=1613661&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcCtx.java Sat Jul 26 15:39:55 2014
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.parse.OpParseContext;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+
+/**
+ * This class implements the processor context for Constant Propagate.
+ * 
+ * ConstantPropagateProcCtx keeps track of propagated constants in a column->const map for each
+ * operator, enabling constants to be revolved across operators.
+ */
+public class ConstantPropagateProcCtx implements NodeProcessorCtx {
+
+  private static final org.apache.commons.logging.Log LOG = LogFactory
+      .getLog(ConstantPropagateProcCtx.class);
+
+  private final Map<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>> opToConstantExprs;
+  private final Map<Operator<? extends OperatorDesc>, OpParseContext> opToParseCtx;
+  private final List<Operator<? extends Serializable>> opToDelete;
+
+  public ConstantPropagateProcCtx(Map<Operator<? extends OperatorDesc>, OpParseContext> opToParseCtx) {
+    opToConstantExprs =
+        new HashMap<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>>();
+    opToDelete = new ArrayList<Operator<? extends Serializable>>();
+    this.opToParseCtx = opToParseCtx;
+  }
+
+  public Map<Operator<? extends Serializable>, Map<ColumnInfo, ExprNodeDesc>> getOpToConstantExprs() {
+    return opToConstantExprs;
+  }
+
+
+  public Map<Operator<? extends OperatorDesc>, OpParseContext> getOpToParseCtxMap() {
+    return opToParseCtx;
+  }
+
+  /**
+   * Resolve a ColumnInfo based on given RowResolver.
+   * 
+   * @param ci
+   * @param rr
+   * @param parentRR 
+   * @return
+   * @throws SemanticException
+   */
+  private ColumnInfo resolve(ColumnInfo ci, RowResolver rr, RowResolver parentRR)
+      throws SemanticException {
+    // Resolve new ColumnInfo from <tableAlias, alias>
+    String alias = ci.getAlias();
+    if (alias == null) {
+      alias = ci.getInternalName();
+    }
+    String tblAlias = ci.getTabAlias();
+    ColumnInfo rci = rr.get(tblAlias, alias);
+    if (rci == null && rr.getRslvMap().size() == 1 && parentRR.getRslvMap().size() == 1) {
+      rci = rr.get(null, alias);
+    }
+    if (rci == null) {
+      return null;
+    }
+    String[] tmp = rr.reverseLookup(rci.getInternalName());
+    rci.setTabAlias(tmp[0]);
+    rci.setAlias(tmp[1]);
+    LOG.debug("Resolved "
+        + ci.getTabAlias() + "." + ci.getAlias() + " as "
+        + rci.getTabAlias() + "." + rci.getAlias() + " with rr: " + rr);
+    return rci;
+  }
+
+  /**
+   * Get propagated constant map from parents.
+   * 
+   * Traverse all parents of current operator, if there is propagated constant (determined by
+   * assignment expression like column=constant value), resolve the column using RowResolver and add
+   * it to current constant map.
+   * 
+   * @param op
+   *        operator getting the propagated constants.
+   * @return map of ColumnInfo to ExprNodeDesc. The values of that map must be either
+   *         ExprNodeConstantDesc or ExprNodeNullDesc.
+   */
+  public Map<ColumnInfo, ExprNodeDesc> getPropagatedConstants(
+      Operator<? extends Serializable> op) {
+    Map<ColumnInfo, ExprNodeDesc> constants = new HashMap<ColumnInfo, ExprNodeDesc>();
+    OpParseContext parseCtx = opToParseCtx.get(op);
+    if (parseCtx == null) {
+      return constants;
+    }
+    RowResolver rr = parseCtx.getRowResolver();
+    LOG.debug("Getting constants of op:" + op + " with rr:" + rr);
+    
+    try {
+      if (op.getParentOperators() == null) {
+        return constants;
+      }
+
+      if (op instanceof UnionOperator) {
+        String alias = (String) rr.getRslvMap().keySet().toArray()[0];
+        // find intersection
+        Map<ColumnInfo, ExprNodeDesc> intersection = null;
+        for (Operator<?> parent : op.getParentOperators()) {
+          Map<ColumnInfo, ExprNodeDesc> unionConst = opToConstantExprs.get(parent);
+          LOG.debug("Constant of op " + parent.getOperatorId() + " " + unionConst);
+          if (intersection == null) {
+            intersection = new HashMap<ColumnInfo, ExprNodeDesc>();
+            for (Entry<ColumnInfo, ExprNodeDesc> e : unionConst.entrySet()) {
+              ColumnInfo ci = new ColumnInfo(e.getKey());
+              ci.setTabAlias(alias);
+              intersection.put(ci, e.getValue());
+            }
+          } else {
+            Iterator<Entry<ColumnInfo, ExprNodeDesc>> itr = intersection.entrySet().iterator();
+            while (itr.hasNext()) {
+              Entry<ColumnInfo, ExprNodeDesc> e = itr.next();
+              boolean found = false;
+              for (Entry<ColumnInfo, ExprNodeDesc> f : opToConstantExprs.get(parent).entrySet()) {
+                if (e.getKey().getInternalName().equals(f.getKey().getInternalName())) {
+                  if (e.getValue().isSame(f.getValue())) {
+                    found = true;
+                  }
+                  break;
+                }
+              }
+              if (!found) {
+                itr.remove();
+              }
+            }
+          }
+          if (intersection.isEmpty()) {
+            return intersection;
+          }
+        }
+        LOG.debug("Propagated union constants:" + intersection);
+        return intersection;
+      }
+
+      for (Operator<? extends Serializable> parent : op.getParentOperators()) {
+        Map<ColumnInfo, ExprNodeDesc> c = opToConstantExprs.get(parent);
+        for (Entry<ColumnInfo, ExprNodeDesc> e : c.entrySet()) {
+          ColumnInfo ci = e.getKey();
+          ColumnInfo rci = null;
+          ExprNodeDesc constant = e.getValue();
+          rci = resolve(ci, rr, opToParseCtx.get(parent).getRowResolver());
+          if (rci != null) {
+            constants.put(rci, constant);
+          } else {
+            LOG.debug("Can't resolve " + ci.getTabAlias() + "." + ci.getAlias() + " from rr:"
+                + rr);
+          }
+
+        }
+
+      }
+      LOG.debug("Offerring constants " + constants.keySet()
+          + " to operator " + op.toString());
+      return constants;
+    } catch (SemanticException e) {
+      LOG.error(e.getMessage(), e);
+      throw new RuntimeException(e);
+    }
+  }
+
+  public RowResolver getRowResolver(Operator<? extends Serializable> op) {
+    OpParseContext parseCtx = opToParseCtx.get(op);
+    if (parseCtx == null) {
+      return null;
+    }
+    return parseCtx.getRowResolver();
+  }
+
+  public void addOpToDelete(Operator<? extends Serializable> op) {
+    opToDelete.add(op);
+  }
+
+  public List<Operator<? extends Serializable>> getOpToDelete() {
+    return opToDelete;
+  }
+}



Mime
View raw message