hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1600719 [1/40] - in /hive/trunk: contrib/src/test/results/clientpositive/ hbase-handler/src/test/results/positive/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ ql/src/java/org/apache/h...
Date Thu, 05 Jun 2014 18:28:15 GMT
Author: hashutosh
Date: Thu Jun  5 18:28:07 2014
New Revision: 1600719

URL: http://svn.apache.org/r1600719
Log:
HIVE-4867 : Deduplicate columns appearing in both the key list and value list of ReduceSinkOperator (Navis via Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java
Modified:
    hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
    hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
    hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcCtx.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
    hive/trunk/ql/src/test/queries/clientpositive/bucketmapjoin6.q
    hive/trunk/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
    hive/trunk/ql/src/test/results/clientpositive/ambiguous_col.q.out
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_join.q.out
    hive/trunk/ql/src/test/results/clientpositive/annotate_stats_select.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join0.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join13.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join15.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join18.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join18_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join2.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join20.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join21.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join23.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join28.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join30.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join31.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join6.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join7.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join_reordering_values.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_smb_mapjoin_14.q.out
    hive/trunk/ql/src/test/results/clientpositive/auto_sortmerge_join_9.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucket5.q.out
    hive/trunk/ql/src/test/results/clientpositive/bucketmapjoin6.q.out
    hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
    hive/trunk/ql/src/test/results/clientpositive/column_access_stats.q.out
    hive/trunk/ql/src/test/results/clientpositive/columnarserde_create_shortcut.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer1.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer10.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer11.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer12.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer13.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer14.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer15.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer2.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer4.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer5.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer8.q.out
    hive/trunk/ql/src/test/results/clientpositive/correlationoptimizer9.q.out
    hive/trunk/ql/src/test/results/clientpositive/count.q.out
    hive/trunk/ql/src/test/results/clientpositive/create_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/cross_join.q.out
    hive/trunk/ql/src/test/results/clientpositive/cross_product_check_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/ctas.q.out
    hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
    hive/trunk/ql/src/test/results/clientpositive/ctas_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/decimal_udf.q.out
    hive/trunk/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out
    hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out
    hive/trunk/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out
    hive/trunk/ql/src/test/results/clientpositive/escape_clusterby1.q.out
    hive/trunk/ql/src/test/results/clientpositive/escape_distributeby1.q.out
    hive/trunk/ql/src/test/results/clientpositive/escape_orderby1.q.out
    hive/trunk/ql/src/test/results/clientpositive/escape_sortby1.q.out
    hive/trunk/ql/src/test/results/clientpositive/explain_logical.q.out
    hive/trunk/ql/src/test/results/clientpositive/explain_rearrange.q.out
    hive/trunk/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby1_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby2_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby2_map_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby3_map_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby7_noskew_multi_single_reducer.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_complex_types_multi_single_reducer.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_distinct_samekey.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_grouping_sets4.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_map_ppr_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_multi_single_reducer.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_multi_single_reducer2.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_multi_single_reducer3.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_position.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_resolution.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_11.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_1_23.q.out
    hive/trunk/ql/src/test/results/clientpositive/groupby_sort_skew_1_23.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
    hive/trunk/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
    hive/trunk/ql/src/test/results/clientpositive/infer_bucket_sort_num_buckets.q.out
    hive/trunk/ql/src/test/results/clientpositive/innerjoin.q.out
    hive/trunk/ql/src/test/results/clientpositive/input11_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/input14.q.out
    hive/trunk/ql/src/test/results/clientpositive/input14_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/input17.q.out
    hive/trunk/ql/src/test/results/clientpositive/input18.q.out
    hive/trunk/ql/src/test/results/clientpositive/input1_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/input20.q.out
    hive/trunk/ql/src/test/results/clientpositive/input21.q.out
    hive/trunk/ql/src/test/results/clientpositive/input22.q.out
    hive/trunk/ql/src/test/results/clientpositive/input25.q.out
    hive/trunk/ql/src/test/results/clientpositive/input26.q.out
    hive/trunk/ql/src/test/results/clientpositive/input33.q.out
    hive/trunk/ql/src/test/results/clientpositive/input3_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/input4_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/input5.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_columnarserde.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_lazyserde.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part10.q.out
    hive/trunk/ql/src/test/results/clientpositive/input_part7.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert1_overwrite_partitions.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert_into1.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert_into2.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert_into3.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert_into4.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert_into5.q.out
    hive/trunk/ql/src/test/results/clientpositive/insert_into6.q.out
    hive/trunk/ql/src/test/results/clientpositive/join0.q.out
    hive/trunk/ql/src/test/results/clientpositive/join1.q.out
    hive/trunk/ql/src/test/results/clientpositive/join10.q.out
    hive/trunk/ql/src/test/results/clientpositive/join11.q.out
    hive/trunk/ql/src/test/results/clientpositive/join12.q.out
    hive/trunk/ql/src/test/results/clientpositive/join13.q.out
    hive/trunk/ql/src/test/results/clientpositive/join14.q.out
    hive/trunk/ql/src/test/results/clientpositive/join14_hadoop20.q.out
    hive/trunk/ql/src/test/results/clientpositive/join15.q.out
    hive/trunk/ql/src/test/results/clientpositive/join16.q.out
    hive/trunk/ql/src/test/results/clientpositive/join17.q.out
    hive/trunk/ql/src/test/results/clientpositive/join18.q.out
    hive/trunk/ql/src/test/results/clientpositive/join18_multi_distinct.q.out
    hive/trunk/ql/src/test/results/clientpositive/join19.q.out
    hive/trunk/ql/src/test/results/clientpositive/join2.q.out
    hive/trunk/ql/src/test/results/clientpositive/join20.q.out
    hive/trunk/ql/src/test/results/clientpositive/join21.q.out
    hive/trunk/ql/src/test/results/clientpositive/join22.q.out
    hive/trunk/ql/src/test/results/clientpositive/join23.q.out
    hive/trunk/ql/src/test/results/clientpositive/join29.q.out
    hive/trunk/ql/src/test/results/clientpositive/join3.q.out
    hive/trunk/ql/src/test/results/clientpositive/join31.q.out
    hive/trunk/ql/src/test/results/clientpositive/join32.q.out
    hive/trunk/ql/src/test/results/clientpositive/join32_lessSize.q.out
    hive/trunk/ql/src/test/results/clientpositive/join33.q.out
    hive/trunk/ql/src/test/results/clientpositive/join4.q.out
    hive/trunk/ql/src/test/results/clientpositive/join40.q.out
    hive/trunk/ql/src/test/results/clientpositive/join41.q.out
    hive/trunk/ql/src/test/results/clientpositive/join5.q.out
    hive/trunk/ql/src/test/results/clientpositive/join6.q.out
    hive/trunk/ql/src/test/results/clientpositive/join7.q.out
    hive/trunk/ql/src/test/results/clientpositive/join8.q.out
    hive/trunk/ql/src/test/results/clientpositive/join9.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_alt_syntax.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_3.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_4.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual1.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual2.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual3.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_cond_pushdown_unqual4.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_filters_overlap.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_hive_626.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_merging.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_nullsafe.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_rc.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_reorder.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_reorder2.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_reorder3.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_star.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_thrift.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out
    hive/trunk/ql/src/test/results/clientpositive/join_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out
    hive/trunk/ql/src/test/results/clientpositive/lateral_view_noalias.q.out
    hive/trunk/ql/src/test/results/clientpositive/limit_pushdown.q.out
    hive/trunk/ql/src/test/results/clientpositive/limit_pushdown_negative.q.out
    hive/trunk/ql/src/test/results/clientpositive/lineage1.q.out
    hive/trunk/ql/src/test/results/clientpositive/load_dyn_part14.q.out
    hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapjoin_filter_on_outerjoin.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapjoin_test_outer.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce1.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce2.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce3.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce4.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce5.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce6.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce7.q.out
    hive/trunk/ql/src/test/results/clientpositive/mapreduce8.q.out
    hive/trunk/ql/src/test/results/clientpositive/merge3.q.out
    hive/trunk/ql/src/test/results/clientpositive/merge4.q.out
    hive/trunk/ql/src/test/results/clientpositive/mergejoins.q.out
    hive/trunk/ql/src/test/results/clientpositive/mergejoins_mixed.q.out
    hive/trunk/ql/src/test/results/clientpositive/multiMapJoin1.q.out
    hive/trunk/ql/src/test/results/clientpositive/multiMapJoin2.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert_gby.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert_gby2.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert_gby3.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_insert_move_tasks_share_dependencies.q.out
    hive/trunk/ql/src/test/results/clientpositive/multi_join_union.q.out
    hive/trunk/ql/src/test/results/clientpositive/no_hooks.q.out
    hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch.q.out
    hive/trunk/ql/src/test/results/clientpositive/nonreserved_keywords_insert_into1.q.out
    hive/trunk/ql/src/test/results/clientpositive/optional_outer.q.out
    hive/trunk/ql/src/test/results/clientpositive/orc_createas1.q.out
    hive/trunk/ql/src/test/results/clientpositive/orc_predicate_pushdown.q.out
    hive/trunk/ql/src/test/results/clientpositive/order.q.out
    hive/trunk/ql/src/test/results/clientpositive/order2.q.out
    hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/parallel_orderby.q.out
    hive/trunk/ql/src/test/results/clientpositive/pcr.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_clusterby.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_gby_join.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join2.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join3.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_join_filter.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_multi_insert.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join1.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join2.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join3.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join4.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_outer_join5.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_random.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_repeated_alias.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_udf_case.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_union_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/ppd_vc.q.out
    hive/trunk/ql/src/test/results/clientpositive/push_or.q.out
    hive/trunk/ql/src/test/results/clientpositive/quotedid_skew.q.out
    hive/trunk/ql/src/test/results/clientpositive/rcfile_null_value.q.out
    hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate.q.out
    hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate_exclude_join.q.out
    hive/trunk/ql/src/test/results/clientpositive/reduce_deduplicate_extended.q.out
    hive/trunk/ql/src/test/results/clientpositive/regex_col.q.out
    hive/trunk/ql/src/test/results/clientpositive/regexp_extract.q.out
    hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample10.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample6.q.out
    hive/trunk/ql/src/test/results/clientpositive/sample8.q.out
    hive/trunk/ql/src/test/results/clientpositive/script_pipe.q.out
    hive/trunk/ql/src/test/results/clientpositive/select_as_omitted.q.out
    hive/trunk/ql/src/test/results/clientpositive/select_transform_hint.q.out
    hive/trunk/ql/src/test/results/clientpositive/semijoin.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoin.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoin_noskew.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt1.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt10.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt11.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt12.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt13.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt14.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt15.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt16.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt17.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt19.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt2.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt20.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt3.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt4.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt5.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt6.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt7.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt8.q.out
    hive/trunk/ql/src/test/results/clientpositive/skewjoinopt9.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_14.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_17.q.out
    hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_25.q.out
    hive/trunk/ql/src/test/results/clientpositive/sort.q.out
    hive/trunk/ql/src/test/results/clientpositive/subq_where_serialization.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_exists.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_exists_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_in.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_in_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_multiinsert.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notexists.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notexists_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notin.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out
    hive/trunk/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out
    hive/trunk/ql/src/test/results/clientpositive/symlink_text_input_format.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/auto_join0.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/bucket_map_join_tez1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/bucket_map_join_tez2.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/count.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/cross_join.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/ctas.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/filter_join_breaktask.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/insert_into1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/insert_into2.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/join0.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/join1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/mapjoin_decimal.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/mapreduce1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/mapreduce2.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/mrr.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/ptf.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/script_pipe.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/scriptfile1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/subquery_exists.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/subquery_in.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/tez_dml.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/tez_join_tests.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/tez_joins_explain.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/tez_schema_evolution.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/tez_union.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/transform_ppr1.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/transform_ppr2.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/union3.q.out
    hive/trunk/ql/src/test/results/clientpositive/transform_ppr1.q.out
    hive/trunk/ql/src/test/results/clientpositive/transform_ppr2.q.out
    hive/trunk/ql/src/test/results/clientpositive/type_widening.q.out
    hive/trunk/ql/src/test/results/clientpositive/udf_case_column_pruning.q.out
    hive/trunk/ql/src/test/results/clientpositive/udtf_explode.q.out
    hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
    hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out
    hive/trunk/ql/src/test/results/clientpositive/union20.q.out
    hive/trunk/ql/src/test/results/clientpositive/union22.q.out
    hive/trunk/ql/src/test/results/clientpositive/union23.q.out
    hive/trunk/ql/src/test/results/clientpositive/union24.q.out
    hive/trunk/ql/src/test/results/clientpositive/union26.q.out
    hive/trunk/ql/src/test/results/clientpositive/union3.q.out
    hive/trunk/ql/src/test/results/clientpositive/union32.q.out
    hive/trunk/ql/src/test/results/clientpositive/union34.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_ppr.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_remove_23.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_remove_25.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_top_level.q.out
    hive/trunk/ql/src/test/results/clientpositive/union_view.q.out
    hive/trunk/ql/src/test/results/clientpositive/vector_between_in.q.out
    hive/trunk/ql/src/test/results/clientpositive/vector_non_string_partition.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorization_div0.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorization_limit.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorization_part_project.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorized_context.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorized_distinct_gby.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorized_shufflejoin.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
    hive/trunk/ql/src/test/results/compiler/plan/case_sensitivity.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/cast1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/groupby1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/groupby2.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/groupby3.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/groupby4.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/groupby5.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/groupby6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input2.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input3.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input7.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input8.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input9.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input_part1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input_testxpath.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample2.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample3.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample4.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample5.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/sample7.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/subq.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf1.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf4.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf6.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf_case.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/udf_when.q.xml
    hive/trunk/ql/src/test/results/compiler/plan/union.q.xml

Modified: hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out Thu Jun  5 18:28:07 2014
@@ -67,9 +67,10 @@ STAGE PLANS:
                       key expressions: _col0 (type: string), _col1 (type: string)
                       sort order: ++
                       Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
-                      value expressions: _col0 (type: string), _col1 (type: string)
       Reduce Operator Tree:
-        Extract
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+          outputColumnNames: _col0, _col1
           Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false

Modified: hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out (original)
+++ hive/trunk/contrib/src/test/results/clientpositive/udf_row_sequence.q.out Thu Jun  5 18:28:07 2014
@@ -53,20 +53,17 @@ STAGE PLANS:
                 key expressions: _col0 (type: string)
                 sort order: +
                 Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-                value expressions: _col0 (type: string)
       Reduce Operator Tree:
-        Extract
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: string), row_sequence() (type: bigint)
+          outputColumnNames: _col0, _col1
           Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-          Select Operator
-            expressions: _col0 (type: string), row_sequence() (type: bigint)
-            outputColumnNames: _col0, _col1
-            Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-            File Output Operator
-              compressed: false
-              table:
-                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                  serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-2
     Map Reduce
@@ -76,9 +73,11 @@ STAGE PLANS:
               key expressions: _col1 (type: bigint)
               sort order: +
               Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col0 (type: string), _col1 (type: bigint)
+              value expressions: _col0 (type: string)
       Reduce Operator Tree:
-        Extract
+        Select Operator
+          expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: bigint)
+          outputColumnNames: _col0, _col1
           Statistics: Num rows: 58 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false

Modified: hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out (original)
+++ hive/trunk/hbase-handler/src/test/results/positive/hbase_queries.q.out Thu Jun  5 18:28:07 2014
@@ -168,9 +168,10 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               sort order: ++
               Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
-              value expressions: _col0 (type: string), _col1 (type: string)
       Reduce Operator Tree:
-        Extract
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+          outputColumnNames: _col0, _col1
           Statistics: Num rows: 31 Data size: 6393 Basic stats: COMPLETE Column stats: NONE
           Limit
             Number of rows: 20
@@ -273,7 +274,7 @@ STAGE PLANS:
                   sort order: +
                   Map-reduce partition columns: _col0 (type: int)
                   Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                  value expressions: _col0 (type: int), _col1 (type: string)
+                  value expressions: _col1 (type: string)
           TableScan
             alias: hbase_table_1
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -295,7 +296,7 @@ STAGE PLANS:
                Inner Join 0 to 1
           condition expressions:
             0 
-            1 {VALUE._col0} {VALUE._col1}
+            1 {KEY.reducesinkkey0} {VALUE._col0}
           outputColumnNames: _col2, _col3
           Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
           Select Operator
@@ -317,9 +318,10 @@ STAGE PLANS:
               key expressions: _col0 (type: int), _col1 (type: string)
               sort order: ++
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-              value expressions: _col0 (type: int), _col1 (type: string)
       Reduce Operator Tree:
-        Extract
+        Select Operator
+          expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string)
+          outputColumnNames: _col0, _col1
           Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
           File Output Operator
             compressed: false

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java Thu Jun  5 18:28:07 2014
@@ -23,16 +23,11 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
 import org.apache.hadoop.hive.ql.exec.persistence.RowContainer;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-
 
 public abstract class AbstractMapJoinOperator <T extends MapJoinDesc> extends CommonJoinOperator<T> implements
     Serializable {
@@ -46,10 +41,6 @@ public abstract class AbstractMapJoinOpe
    * The ObjectInspectors for the join inputs's join keys.
    */
   protected transient List<ObjectInspector>[] joinKeysObjectInspectors;
-  /**
-   * The standard ObjectInspectors for the join inputs's join keys.
-   */
-  protected transient List<ObjectInspector>[] joinKeysStandardObjectInspectors;
 
   protected transient byte posBigTable = -1; // pos of driver alias
 
@@ -70,11 +61,6 @@ public abstract class AbstractMapJoinOpe
   @Override
   @SuppressWarnings("unchecked")
   protected void initializeOp(Configuration hconf) throws HiveException {
-    super.initializeOp(hconf);
-
-    numMapRowsRead = 0;
-    firstRow = true;
-
     int tagLen = conf.getTagLength();
 
     joinKeys = new List[tagLen];
@@ -82,8 +68,11 @@ public abstract class AbstractMapJoinOpe
     JoinUtil.populateJoinKeyValue(joinKeys, conf.getKeys(), NOTSKIPBIGTABLE);
     joinKeysObjectInspectors = JoinUtil.getObjectInspectorsFromEvaluators(joinKeys,
         inputObjInspectors,NOTSKIPBIGTABLE, tagLen);
-    joinKeysStandardObjectInspectors = JoinUtil.getStandardObjectInspectors(
-        joinKeysObjectInspectors,NOTSKIPBIGTABLE, tagLen);
+
+    super.initializeOp(hconf);
+
+    numMapRowsRead = 0;
+    firstRow = true;
 
     // all other tables are small, and are cached in the hash table
     posBigTable = (byte) conf.getPosBigTable();
@@ -96,27 +85,25 @@ public abstract class AbstractMapJoinOpe
         !hasFilter(posBigTable), reporter);
     storage[posBigTable] = bigPosRC;
 
-    List<? extends StructField> structFields = ((StructObjectInspector) outputObjInspector)
-        .getAllStructFieldRefs();
-    if (conf.getOutputColumnNames().size() < structFields.size()) {
-      List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
-      for (Byte alias : order) {
-        int sz = conf.getExprs().get(alias).size();
-        List<Integer> retained = conf.getRetainList().get(alias);
-        for (int i = 0; i < sz; i++) {
-          int pos = retained.get(i);
-          structFieldObjectInspectors.add(structFields.get(pos)
-              .getFieldObjectInspector());
-        }
-      }
-      outputObjInspector = ObjectInspectorFactory
-          .getStandardStructObjectInspector(conf.getOutputColumnNames(),
-          structFieldObjectInspectors);
-    }
     initializeChildren(hconf);
   }
 
   @Override
+  protected List<ObjectInspector> getValueObjectInspectors(
+      byte alias, List<ObjectInspector>[] aliasToObjectInspectors) {
+    List<ObjectInspector> inspectors = aliasToObjectInspectors[alias];
+    List<Integer> retained = conf.getRetainList().get(alias);
+    if (inspectors.size() == retained.size()) {
+      return inspectors;
+    }
+    List<ObjectInspector> retainedOIs = new ArrayList<ObjectInspector>();
+    for (int index : retained) {
+      retainedOIs.add(inspectors.get(index));
+    }
+    return retainedOIs;
+  }
+
+  @Override
   public OperatorType getType() {
     return OperatorType.MAPJOIN;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java Thu Jun  5 18:28:07 2014
@@ -198,6 +198,11 @@ public class ColumnInfo implements Seria
   }
 
   @Override
+  public int hashCode() {
+    return internalName.hashCode() + typeName.hashCode();
+  }
+
+  @Override
   public boolean equals(Object obj) {
     if (!(obj instanceof ColumnInfo) || (obj == null)) {
       return false;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Thu Jun  5 18:28:07 2014
@@ -160,14 +160,12 @@ public abstract class CommonJoinOperator
     this.joinFilterObjectInspectors = clone.joinFilterObjectInspectors;
   }
 
-
-  protected static <T extends JoinDesc> ObjectInspector getJoinOutputObjectInspector(
-      Byte[] order, List<ObjectInspector>[] aliasToObjectInspectors,
-      T conf) {
+  private <T extends JoinDesc> ObjectInspector getJoinOutputObjectInspector(
+      Byte[] order, List<ObjectInspector>[] aliasToObjectInspectors, T conf) {
     List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>();
     for (Byte alias : order) {
-      List<ObjectInspector> oiList = aliasToObjectInspectors[alias];
-      if (oiList != null) {
+      List<ObjectInspector> oiList = getValueObjectInspectors(alias, aliasToObjectInspectors);
+      if (oiList != null && !oiList.isEmpty()) {
         structFieldObjectInspectors.addAll(oiList);
       }
     }
@@ -178,6 +176,11 @@ public abstract class CommonJoinOperator
     return joinOutputObjectInspector;
   }
 
+  protected List<ObjectInspector> getValueObjectInspectors(
+      byte alias, List<ObjectInspector>[] aliasToObjectInspectors) {
+    return aliasToObjectInspectors[alias];
+  }
+
   protected Configuration hconf;
 
   @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Thu Jun  5 18:28:07 2014
@@ -34,8 +34,6 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionHandler;
 import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKeyObject;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinPersistableTableContainer;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinEagerRowContainer;
@@ -226,8 +224,13 @@ public class HashTableSinkOperator exten
   public void processOp(Object row, int tag) throws HiveException {
     byte alias = (byte)tag;
     // compute keys and values as StandardObjects. Use non-optimized key (MR).
-    MapJoinKey key = MapJoinKey.readFromRow(null, new MapJoinKeyObject(),
-        row, joinKeys[alias], joinKeysObjectInspectors[alias], true);
+    Object[] currentKey = new Object[joinKeys[alias].size()];
+    for (int keyIndex = 0; keyIndex < joinKeys[alias].size(); ++keyIndex) {
+      currentKey[keyIndex] = joinKeys[alias].get(keyIndex).evaluate(row);
+    }
+    MapJoinKeyObject key = new MapJoinKeyObject();
+    key.readFromRow(currentKey, joinKeysObjectInspectors[alias]);
+
     Object[] value = EMPTY_OBJECT_ARRAY;
     if((hasFilter(alias) && filterMaps[alias].length > 0) || joinValues[alias].size() > 0) {
       value = JoinUtil.computeMapJoinValues(row, joinValues[alias],
@@ -269,8 +272,10 @@ public class HashTableSinkOperator exten
         flushToFile();
       }
       super.closeOp(abort);
+    } catch (HiveException e) {
+      throw e;
     } catch (Exception e) {
-      LOG.error("Error generating side-table", e);
+      throw new HiveException(e);
     }
   }
 
@@ -290,7 +295,8 @@ public class HashTableSinkOperator exten
       // get the tmp URI path; it will be a hdfs path if not local mode
       String dumpFilePrefix = conf.getDumpFilePrefix();
       Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName);
-      console.printInfo(Utilities.now() + "\tDump the side-table into file: " + path);
+      console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag +
+          " with group count: " + tableContainer.size() + " into file: " + path);
       // get the hashtable file and path
       FileSystem fs = path.getFileSystem(hconf);
       ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(fs.create(path), 4096));

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Thu Jun  5 18:28:07 2014
@@ -19,28 +19,29 @@
 package org.apache.hadoop.hive.ql.exec;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.ql.HashTableLoaderFactory;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinRowContainer;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer.ReusableGetAdaptor;
 import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
+import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.ql.plan.api.OperatorType;
-import org.apache.hadoop.hive.serde2.ByteStream.Output;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.util.ReflectionUtils;
 
 /**
@@ -64,6 +65,8 @@ public class MapJoinOperator extends Abs
   private transient boolean hashTblInitedOnce;
   private transient ReusableGetAdaptor[] hashMapRowGetters;
 
+  private UnwrapRowContainer[] unwrapContainer;
+
   public MapJoinOperator() {
   }
 
@@ -88,6 +91,7 @@ public class MapJoinOperator extends Abs
 
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {
+    unwrapContainer = new UnwrapRowContainer[conf.getTagLength()];
     super.initializeOp(hconf);
 
     int tagLen = conf.getTagLength();
@@ -113,6 +117,29 @@ public class MapJoinOperator extends Abs
     }
   }
 
+  @Override
+  protected List<ObjectInspector> getValueObjectInspectors(
+      byte alias, List<ObjectInspector>[] aliasToObjectInspectors) {
+    int[] valueIndex = conf.getValueIndex(alias);
+    if (valueIndex == null) {
+      return super.getValueObjectInspectors(alias, aliasToObjectInspectors);
+    }
+    unwrapContainer[alias] = new UnwrapRowContainer(valueIndex);
+
+    List<ObjectInspector> inspectors = aliasToObjectInspectors[alias];
+
+    int bigPos = conf.getPosBigTable();
+    List<ObjectInspector> valueOI = new ArrayList<ObjectInspector>();
+    for (int i = 0; i < valueIndex.length; i++) {
+      if (valueIndex[i] >= 0) {
+        valueOI.add(joinKeysObjectInspectors[bigPos].get(valueIndex[i]));
+      } else {
+        valueOI.add(inspectors.get(i));
+      }
+    }
+    return valueOI;
+  }
+
   public void generateMapMetaData() throws HiveException, SerDeException {
     // generate the meta data for key
     // index for key is -1
@@ -237,6 +264,10 @@ public class MapJoinOperator extends Abs
             adaptor.setFromOther(firstSetKey);
           }
           MapJoinRowContainer rowContainer = adaptor.getCurrentRows();
+          if (rowContainer != null && unwrapContainer[pos] != null) {
+            Object[] currentKey = adaptor.getCurrentKey();
+            rowContainer = unwrapContainer[pos].setInternal(rowContainer, currentKey);
+          }
           // there is no join-value or join-key has all null elements
           if (rowContainer == null || adaptor.hasAnyNulls(fieldCount, nullsafes)) {
             if (!noOuterJoin) {
@@ -267,7 +298,7 @@ public class MapJoinOperator extends Abs
         }
       }
     } catch (Exception e) {
-      String msg = "Unxpected exception: " + e.getMessage();
+      String msg = "Unexpected exception: " + e.getMessage();
       LOG.error(msg, e);
       throw new HiveException(msg, e);
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Thu Jun  5 18:28:07 2014
@@ -383,7 +383,7 @@ public class MapOperator extends Operato
           if (!onepath.toUri().relativize(fpath.toUri()).equals(fpath.toUri())) {
             children.add(op);
             childrenOpToOpCtxMap.put(op, opCtx);
-            LOG.info("dump " + op.getName() + " "
+            LOG.info("dump " + op + " "
                 + opCtxMap.get(inp).rowObjectInspector.getTypeName());
           }
           current = opCtx;  // just need for TestOperators.testMapOperator

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Thu Jun  5 18:28:07 2014
@@ -32,7 +32,6 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -98,6 +97,11 @@ public abstract class Operator<T extends
 
   private boolean useBucketizedHiveInputFormat;
 
+  // dummy operator (for not increasing seqId)
+  private Operator(String name) {
+    id = name;
+  }
+
   public Operator() {
     id = String.valueOf(seqId.getAndIncrement());
     childOperators = new ArrayList<Operator<? extends OperatorDesc>>();
@@ -334,7 +338,7 @@ public abstract class Operator<T extends
       return;
     }
 
-    LOG.info("Initializing Self " + id + " " + getName());
+    LOG.info("Initializing Self " + this);
 
     if (inputOIs != null) {
       inputObjInspectors = inputOIs;
@@ -1288,4 +1292,14 @@ public abstract class Operator<T extends
   public ObjectInspector getGroupKeyObjectInspector() {
     return groupKeyOI;
   }
+
+  public static Operator createDummy() {
+    return new DummyOperator();
+  }
+
+  private static class DummyOperator extends Operator {
+    public DummyOperator() { super("dummy"); }
+    public void processOp(Object row, int tag) { }
+    public OperatorType getType() { return null; }
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Thu Jun  5 18:28:07 2014
@@ -57,7 +57,7 @@ public class ReduceSinkOperator extends 
     implements Serializable, TopNHash.BinaryCollector {
 
   static {
-    PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases");
+    PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases", "valueIndex");
   }
 
   private static final long serialVersionUID = 1L;
@@ -96,6 +96,8 @@ public class ReduceSinkOperator extends 
   transient String[] inputAliases;  // input aliases of this RS for join (used for PPD)
   private boolean skipTag = false;
 
+  private transient int[] valueIndex; // index for value(+ from keys, - from values)
+
   public void setInputAliases(String[] inputAliases) {
     this.inputAliases = inputAliases;
   }
@@ -470,4 +472,12 @@ public class ReduceSinkOperator extends 
   public void setSkipTag(boolean value) {
     this.skipTag = value;
   }
+
+  public void setValueIndex(int[] valueIndex) {
+    this.valueIndex = valueIndex;
+  }
+
+  public int[] getValueIndex() {
+    return valueIndex;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RowSchema.java Thu Jun  5 18:28:07 2014
@@ -27,7 +27,7 @@ import java.util.ArrayList;
 public class RowSchema implements Serializable {
 
   private static final long serialVersionUID = 1L;
-  private ArrayList<ColumnInfo> signature;
+  private ArrayList<ColumnInfo> signature = new ArrayList<ColumnInfo>();
 
   public RowSchema() {
   }
@@ -52,13 +52,11 @@ public class RowSchema implements Serial
   public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append('(');
-    if (signature != null) {
-      for (ColumnInfo col: signature) {
-        if (sb.length() > 1) {
-          sb.append(',');
-        }
-        sb.append(col.toString());
+    for (ColumnInfo col: signature) {
+      if (sb.length() > 1) {
+        sb.append(',');
       }
+      sb.append(col.toString());
     }
     sb.append(')');
     return sb.toString();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Thu Jun  5 18:28:07 2014
@@ -221,6 +221,13 @@ public final class Utilities {
     }
   }
 
+  public static String removeValueTag(String column) {
+    if (column.startsWith(ReduceField.VALUE + ".")) {
+      return column.substring(6);
+    }
+    return column;
+  }
+
   private Utilities() {
     // prevent instantiation
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java Thu Jun  5 18:28:07 2014
@@ -23,7 +23,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.Map.Entry;
 import java.util.Set;
 
@@ -36,9 +35,7 @@ import org.apache.hadoop.hive.ql.exec.ve
 import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.WriteBuffers;
 import org.apache.hadoop.hive.serde2.ByteStream.Output;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.io.BytesWritable;
@@ -143,9 +140,13 @@ public class HashMapWrapper extends Abst
   }
 
   private class GetAdaptor implements ReusableGetAdaptor {
+
+    private Object[] currentKey;
+    private List<ObjectInspector> vectorKeyOIs;
+
     private MapJoinKey key;
     private MapJoinRowContainer currentValue;
-    private Output output;
+    private final Output output = new Output();
     private boolean isFirstKey = true;
 
     public GetAdaptor(MapJoinKey key) {
@@ -155,11 +156,17 @@ public class HashMapWrapper extends Abst
     @Override
     public void setFromVector(VectorHashKeyWrapper kw, VectorExpressionWriter[] keyOutputWriters,
         VectorHashKeyWrapperBatch keyWrapperBatch) throws HiveException {
-      if (output == null) {
-        output = new Output(0);
+      if (currentKey == null) {
+        currentKey = new Object[keyOutputWriters.length];
+        vectorKeyOIs = new ArrayList<ObjectInspector>();
+        for (int i = 0; i < keyOutputWriters.length; i++) {
+          vectorKeyOIs.add(keyOutputWriters[i].getObjectInspector());
+        }
       }
-      key = MapJoinKey.readFromVector(
-          output, key, kw, keyOutputWriters, keyWrapperBatch, !isFirstKey);
+      for (int i = 0; i < keyOutputWriters.length; i++) {
+        currentKey[i] = keyWrapperBatch.getWritableKeyValue(kw, i, keyOutputWriters[i]);
+      }
+      key =  MapJoinKey.readFromVector(output, key, currentKey, vectorKeyOIs, !isFirstKey);
       isFirstKey = false;
       this.currentValue = mHash.get(key);
     }
@@ -167,10 +174,13 @@ public class HashMapWrapper extends Abst
     @Override
     public void setFromRow(Object row, List<ExprNodeEvaluator> fields,
         List<ObjectInspector> ois) throws HiveException {
-      if (output == null) {
-        output = new Output(0);
+      if (currentKey == null) {
+        currentKey = new Object[fields.size()];
+      }
+      for (int keyIndex = 0; keyIndex < fields.size(); ++keyIndex) {
+        currentKey[keyIndex] = fields.get(keyIndex).evaluate(row);
       }
-      key = MapJoinKey.readFromRow(output, key, row, fields, ois, !isFirstKey);
+      key = MapJoinKey.readFromRow(output, key, currentKey, ois, !isFirstKey);
       isFirstKey = false;
       this.currentValue = mHash.get(key);
     }
@@ -193,6 +203,11 @@ public class HashMapWrapper extends Abst
     public MapJoinRowContainer getCurrentRows() {
       return currentValue;
     }
+
+    @Override
+    public Object[] getCurrentKey() {
+      return currentKey;
+    }
   }
 
   @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java Thu Jun  5 18:28:07 2014
@@ -9,7 +9,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.debug.Utils;
 import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
 import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapper;
 import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
@@ -293,9 +292,18 @@ public class MapJoinBytesTableContainer 
   /** Implementation of ReusableGetAdaptor that has Output for key serialization; row
    * container is also created once and reused for every row. */
   private class GetAdaptor implements ReusableGetAdaptor {
-    private ReusableRowContainer currentValue = new ReusableRowContainer();
-    private Output output;
+
+    private Object[] currentKey;
     private boolean[] nulls;
+    private List<ObjectInspector> vectorKeyOIs;
+
+    private final ReusableRowContainer currentValue;
+    private final Output output;
+
+    public GetAdaptor() {
+      currentValue = new ReusableRowContainer();
+      output = new Output();
+    }
 
     @Override
     public void setFromVector(VectorHashKeyWrapper kw,
@@ -303,16 +311,20 @@ public class MapJoinBytesTableContainer 
         VectorHashKeyWrapperBatch keyWrapperBatch) throws HiveException {
       if (nulls == null) {
         nulls = new boolean[keyOutputWriters.length];
+        currentKey = new Object[keyOutputWriters.length];
+        vectorKeyOIs = new ArrayList<ObjectInspector>();
+        for (int i = 0; i < keyOutputWriters.length; i++) {
+          vectorKeyOIs.add(keyOutputWriters[i].getObjectInspector());
+        }
       } else {
         assert nulls.length == keyOutputWriters.length;
       }
-      try {
-        output = MapJoinKey.serializeVector(
-            output, kw, keyOutputWriters, keyWrapperBatch, nulls, sortableSortOrders);
-      } catch (SerDeException e) {
-        throw new HiveException(e);
+      for (int i = 0; i < keyOutputWriters.length; i++) {
+        currentKey[i] = keyWrapperBatch.getWritableKeyValue(kw, i, keyOutputWriters[i]);
+        nulls[i] = currentKey[i] == null;
       }
-      this.currentValue.setFromOutput(output);
+      currentValue.setFromOutput(
+          MapJoinKey.serializeRow(output, currentKey, vectorKeyOIs, sortableSortOrders));
     }
 
     @Override
@@ -320,28 +332,23 @@ public class MapJoinBytesTableContainer 
         List<ObjectInspector> ois) throws HiveException {
       if (nulls == null) {
         nulls = new boolean[fields.size()];
+        currentKey = new Object[fields.size()];
       }
-      Object[] fieldObjs = new Object[fields.size()];
       for (int keyIndex = 0; keyIndex < fields.size(); ++keyIndex) {
-        fieldObjs[keyIndex] = fields.get(keyIndex).evaluate(row);
-        nulls[keyIndex] = (fieldObjs[keyIndex] == null);
+        currentKey[keyIndex] = fields.get(keyIndex).evaluate(row);
+        nulls[keyIndex] = currentKey[keyIndex] == null;
       }
-      try {
-        output = MapJoinKey.serializeRow(output, fieldObjs, ois, sortableSortOrders);
-      } catch (SerDeException e) {
-        throw new HiveException(e);
-      }
-      this.currentValue.setFromOutput(output);
+      currentValue.setFromOutput(
+          MapJoinKey.serializeRow(output, currentKey, ois, sortableSortOrders));
     }
 
-
     @Override
     public void setFromOther(ReusableGetAdaptor other) {
       assert other instanceof GetAdaptor;
       GetAdaptor other2 = (GetAdaptor)other;
-      output = other2.output;
       nulls = other2.nulls;
-      this.currentValue.setFromOutput(output);
+      currentKey = other2.currentKey;
+      currentValue.setFromOutput(other2.output);
     }
 
     @Override
@@ -359,6 +366,11 @@ public class MapJoinBytesTableContainer 
     public MapJoinRowContainer getCurrentRows() {
       return currentValue.isEmpty() ? null : currentValue;
     }
+
+    @Override
+    public Object[] getCurrentKey() {
+      return currentKey;
+    }
   }
 
   /** Row container that gets and deserializes the rows on demand from bytes provided. */

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java Thu Jun  5 18:28:07 2014
@@ -27,9 +27,6 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.common.type.Decimal128;
-import org.apache.hadoop.hive.ql.debug.Utils;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
 import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapper;
 import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
@@ -38,10 +35,7 @@ import org.apache.hadoop.hive.serde2.Byt
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
 import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
 import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils;
-import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.StringWrapper;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -66,7 +60,7 @@ public abstract class MapJoinKey {
   @SuppressWarnings("deprecation")
   public static MapJoinKey read(Output output, MapJoinKey key,
       MapJoinObjectSerDeContext context, Writable writable, boolean mayReuseKey)
-          throws SerDeException {
+      throws SerDeException, HiveException {
     SerDe serde = context.getSerDe();
     Object obj = serde.deserialize(writable);
     boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
@@ -106,7 +100,7 @@ public abstract class MapJoinKey {
   }
 
   private static byte[] serialize(Output byteStream,
-      Object obj, ObjectInspector oi, boolean checkTypes) throws SerDeException {
+      Object obj, ObjectInspector oi, boolean checkTypes) throws HiveException {
     if (null == obj || !(oi instanceof StructObjectInspector)) {
       return null; // not supported
     }
@@ -141,29 +135,23 @@ public abstract class MapJoinKey {
     return true;
   }
 
-  public static MapJoinKey readFromVector(Output output, MapJoinKey key, VectorHashKeyWrapper kw,
-      VectorExpressionWriter[] keyOutputWriters, VectorHashKeyWrapperBatch keyWrapperBatch,
-      boolean mayReuseKey) throws HiveException {
+  public static MapJoinKey readFromVector(Output output, MapJoinKey key, Object[] keyObject,
+      List<ObjectInspector> keyOIs, boolean mayReuseKey) throws HiveException {
     boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
     if (useOptimized || key == null) {
-      byte[] structBytes = null;
-      try {
-        if (keyOutputWriters.length <= 8) {
-          output = serializeVector(output, kw, keyOutputWriters, keyWrapperBatch, null, null);
-          structBytes = Arrays.copyOf(output.getData(), output.getLength());
-        }
-      } catch (SerDeException e) {
-        throw new HiveException(e);
+      if (keyObject.length <= 8) {
+        output = serializeRow(output, keyObject, keyOIs, null);
+        return MapJoinKeyBytes.fromBytes(key, mayReuseKey,
+            Arrays.copyOf(output.getData(), output.getLength()));
       }
-      if (structBytes != null) {
-        return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
-      } else if (useOptimized) {
+      if (useOptimized) {
         throw new HiveException(
-            "Failed to serialize " + kw + " even though optimized keys are used");
+            "Failed to serialize " + Arrays.toString(keyObject) +
+                " even though optimized keys are used");
       }
     }
     MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
-    result.readFromVector(kw, keyOutputWriters, keyWrapperBatch);
+    result.setKeyObjects(keyObject);
     return result;
   }
 
@@ -189,37 +177,28 @@ public abstract class MapJoinKey {
     return serializeRow(byteStream, fieldData, fieldOis, sortableSortOrders);
   }
 
-  public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object row,
-      List<ExprNodeEvaluator> fields, List<ObjectInspector> keyFieldsOI, boolean mayReuseKey)
-          throws HiveException {
-    Object[] fieldObjs = new Object[fields.size()];
-    for (int keyIndex = 0; keyIndex < fields.size(); ++keyIndex) {
-      fieldObjs[keyIndex] = fields.get(keyIndex).evaluate(row);
-    }
+  public static MapJoinKey readFromRow(Output output, MapJoinKey key, Object[] keyObject,
+      List<ObjectInspector> keyFieldsOI, boolean mayReuseKey) throws HiveException {
     boolean useOptimized = useOptimizedKeyBasedOnPrev(key);
     if (useOptimized || key == null) {
-      try {
-        byte[] structBytes = null;
-        if (fieldObjs.length <= 8) {
-          if (fieldObjs.length == 0) {
-            structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys
-          } else {
-            output = serializeRow(output, fieldObjs, keyFieldsOI, null);
-            structBytes = Arrays.copyOf(output.getData(), output.getLength());
-          }
-        }
-        if (structBytes != null) {
-          return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
-        } else if (useOptimized) {
-          throw new HiveException(
-              "Failed to serialize " + row + " even though optimized keys are used");
+      if (keyObject.length <= 8) {
+        byte[] structBytes;
+        if (keyObject.length == 0) {
+          structBytes = EMPTY_BYTE_ARRAY; // shortcut for null keys
+        } else {
+          output = serializeRow(output, keyObject, keyFieldsOI, null);
+          structBytes = Arrays.copyOf(output.getData(), output.getLength());
         }
-      } catch (SerDeException ex) {
-        throw new HiveException("Serialization error", ex);
+        return MapJoinKeyBytes.fromBytes(key, mayReuseKey, structBytes);
+      }
+      if (useOptimized) {
+        throw new HiveException(
+            "Failed to serialize " + Arrays.toString(keyObject) +
+                " even though optimized keys are used");
       }
     }
     MapJoinKeyObject result = mayReuseKey ? (MapJoinKeyObject)key : new MapJoinKeyObject();
-    result.readFromRow(fieldObjs, keyFieldsOI);
+    result.readFromRow(keyObject, keyFieldsOI);
     return result;
   }
 
@@ -231,18 +210,22 @@ public abstract class MapJoinKey {
    * @param byteStream Output to reuse. Can be null, in that case a new one would be created.
    */
   public static Output serializeRow(Output byteStream, Object[] fieldData,
-      List<ObjectInspector> fieldOis, boolean[] sortableSortOrders) throws SerDeException {
+      List<ObjectInspector> fieldOis, boolean[] sortableSortOrders) throws HiveException {
     if (byteStream == null) {
       byteStream = new Output();
     } else {
       byteStream.reset();
     }
-    if (fieldData.length == 0) {
-      byteStream.reset();
-    } else if (sortableSortOrders == null) {
-      LazyBinarySerDe.serializeStruct(byteStream, fieldData, fieldOis);
-    } else {
-      BinarySortableSerDe.serializeStruct(byteStream, fieldData, fieldOis, sortableSortOrders);
+    try {
+      if (fieldData.length == 0) {
+        byteStream.reset();
+      } else if (sortableSortOrders == null) {
+        LazyBinarySerDe.serializeStruct(byteStream, fieldData, fieldOis);
+      } else {
+        BinarySortableSerDe.serializeStruct(byteStream, fieldData, fieldOis, sortableSortOrders);
+      }
+    } catch (SerDeException e) {
+      throw new HiveException("Serialization error", e);
     }
     return byteStream;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKeyObject.java Thu Jun  5 18:28:07 2014
@@ -46,10 +46,19 @@ public class MapJoinKeyObject extends Ma
   public MapJoinKeyObject(Object[] key) {
     this.key = key;
   }
+
   public MapJoinKeyObject() {
     this(EMPTY_OBJECT_ARRAY);
   }
 
+  public Object[] getKeyObjects() {
+    return key;
+  }
+
+  public void setKeyObjects(Object[] key) {
+    this.key = key;
+  }
+
   public int getKeyLength() {
     return key.length;
   }
@@ -118,7 +127,7 @@ public class MapJoinKeyObject extends Ma
     container.write(out);
   }
 
-  protected void readFromRow(Object[] fieldObjs, List<ObjectInspector> keyFieldsOI)
+  public void readFromRow(Object[] fieldObjs, List<ObjectInspector> keyFieldsOI)
       throws HiveException {
     if (key == null || key.length != fieldObjs.length) {
       key = new Object[fieldObjs.length];

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java Thu Jun  5 18:28:07 2014
@@ -65,6 +65,11 @@ public interface MapJoinTableContainer {
      * @return The container w/the rows corresponding to a key set via a previous set... call.
      */
     MapJoinRowContainer getCurrentRows();
+
+    /**
+     * @return key
+     */
+    Object[] getCurrentKey();
   }
 
   /**

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java?rev=1600719&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/UnwrapRowContainer.java Thu Jun  5 18:28:07 2014
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.persistence;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.SerDeException;
+
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+public class UnwrapRowContainer
+    implements MapJoinRowContainer, AbstractRowContainer.RowIterator<List<Object>> {
+
+  private final int[] index;
+  private final List<Object> unwrapped;
+
+  private transient Object[] currentKey;
+  private transient MapJoinRowContainer internal;
+
+  private transient RowIterator<List<Object>> iterator;
+
+  public UnwrapRowContainer(int[] valueIndex)  {
+    index = valueIndex;
+    unwrapped = new ArrayList<Object>();
+  }
+
+  public MapJoinRowContainer setInternal(MapJoinRowContainer internal, Object[] currentKey) {
+    this.internal = internal;
+    this.currentKey = currentKey;
+    return this;
+  }
+
+  @Override
+  public List<Object> first() throws HiveException {
+    iterator = internal.rowIter();
+    return unwrap(iterator.first());
+  }
+
+  @Override
+  public List<Object> next() throws HiveException {
+    return unwrap(iterator.next());
+  }
+
+  private List<Object> unwrap(List<Object> values) {
+    if (values == null) {
+      return null;
+    }
+    unwrapped.clear();
+    for (int i = 0; i < index.length; i++) {
+      if (index[i] >= 0) {
+        unwrapped.add(currentKey[index[i]]);
+      } else {
+        int vindex = -index[i] - 1;
+        if (vindex < values.size()) {
+          unwrapped.add(values.get(vindex));
+        }
+      }
+    }
+    return unwrapped;
+  }
+
+  @Override
+  public RowIterator<List<Object>> rowIter() throws HiveException {
+    return this;
+  }
+
+  @Override
+  public void addRow(List<Object> t) throws HiveException {
+    internal.addRow(t);
+  }
+
+  @Override
+  public int rowCount() throws HiveException {
+    return internal.rowCount();
+  }
+
+  @Override
+  public void clearRows() throws HiveException {
+    internal.clearRows();
+  }
+
+  @Override
+  public byte getAliasFilter() throws HiveException {
+    return internal.getAliasFilter();
+  }
+
+  @Override
+  public MapJoinRowContainer copy() throws HiveException {
+    internal = internal.copy();
+    return this;
+  }
+
+  @Override
+  public void addRow(Object[] value) throws HiveException {
+    internal.addRow(value);
+  }
+
+  @Override
+  public void write(MapJoinObjectSerDeContext valueContext, ObjectOutputStream out)
+      throws IOException, SerDeException {
+    internal.write(valueContext, out);
+  }
+}

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Thu Jun  5 18:28:07 2014
@@ -375,6 +375,7 @@ public final class ColumnPrunerProcFacto
         Object... nodeOutputs) throws SemanticException {
       ReduceSinkOperator op = (ReduceSinkOperator) nd;
       ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
+      RowResolver resolver = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
       ReduceSinkDesc conf = op.getConf();
 
       List<String> colLists = new ArrayList<String>();
@@ -395,35 +396,20 @@ public final class ColumnPrunerProcFacto
         childCols = cppCtx.getPrunedColList(child);
 
       }
+      List<ExprNodeDesc> valCols = conf.getValueCols();
+      List<String> valColNames = conf.getOutputValueColumnNames();
+
       if (childCols != null) {
-        /*
-         * in the case of count(or sum) distinct if we are not able to map
-         * a parameter column references back to the ReduceSink value columns
-         * we give up and assume all columns are needed.
-         */
-        boolean hasUnresolvedReference = false;
-        boolean[] flags = new boolean[conf.getValueCols().size()];
+        boolean[] flags = new boolean[valCols.size()];
         Map<String, ExprNodeDesc> exprMap = op.getColumnExprMap();
+
         for (String childCol : childCols) {
-          ExprNodeDesc desc = exprMap.get(childCol);
-          int index = conf.getValueCols().indexOf(desc);
+          int index = valColNames.indexOf(Utilities.removeValueTag(childCol));
           if (index < 0) {
-            hasUnresolvedReference = desc == null || ExprNodeDescUtils.indexOf(desc, conf.getKeyCols()) < 0;
-            if ( hasUnresolvedReference ) {
-              break;
-            }
             continue;
           }
           flags[index] = true;
-          colLists = Utilities.mergeUniqElems(colLists, desc.getCols());
-        }
-        
-        if ( hasUnresolvedReference ) {
-          for (ExprNodeDesc val : conf.getValueCols()) {
-            colLists = Utilities.mergeUniqElems(colLists, val.getCols());
-          }
-          cppCtx.getPrunedColLists().put(op, colLists);
-          return null;
+          colLists = Utilities.mergeUniqElems(colLists, valCols.get(index).getCols());
         }
         
         Collections.sort(colLists);
@@ -434,8 +420,7 @@ public final class ColumnPrunerProcFacto
       
       // Reduce Sink contains the columns needed - no need to aggregate from
       // children
-      ArrayList<ExprNodeDesc> vals = conf.getValueCols();
-      for (ExprNodeDesc val : vals) {
+      for (ExprNodeDesc val : valCols) {
         colLists = Utilities.mergeUniqElems(colLists, val.getCols());
       }
 
@@ -597,10 +582,6 @@ public final class ColumnPrunerProcFacto
 
       // do we need to prune the select operator?
       List<ExprNodeDesc> originalColList = op.getConf().getColList();
-      List<String> columns = new ArrayList<String>();
-      for (ExprNodeDesc expr : originalColList) {
-        Utilities.mergeUniqElems(columns, expr.getCols());
-      }
       // by now, 'prunedCols' are columns used by child operators, and 'columns'
       // are columns used by this select operator.
       List<String> originalOutputColumnNames = conf.getOutputColumnNames();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1600719&r1=1600718&r2=1600719&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Thu Jun  5 18:28:07 2014
@@ -358,7 +358,7 @@ public class ConvertJoinMapJoin implemen
     ParseContext parseContext = context.parseContext;
     MapJoinOperator mapJoinOp = MapJoinProcessor.
       convertJoinOpMapJoinOp(context.conf, parseContext.getOpParseCtx(),
-          joinOp, parseContext.getJoinContext().get(joinOp), bigTablePosition, true);
+          joinOp, parseContext.getJoinContext().get(joinOp), bigTablePosition, true, true);
 
     Operator<? extends OperatorDesc> parentBigTableOp
       = mapJoinOp.getParentOperators().get(bigTablePosition);



Mime
View raw message