hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1608959 - in /hive/trunk: itests/qtest/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/ ql...
Date Tue, 08 Jul 2014 21:48:42 GMT
Author: gunther
Date: Tue Jul  8 21:48:41 2014
New Revision: 1608959

URL: http://svn.apache.org/r1608959
Log:
HIVE-7299: Enable metadata only optimization on Tez (Gunther Hagleitner, reviewed by Vikram
Dixit K)

Added:
    hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries_with_filters.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
Modified:
    hive/trunk/itests/qtest/testconfiguration.properties
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java

Modified: hive/trunk/itests/qtest/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest/testconfiguration.properties?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/itests/qtest/testconfiguration.properties (original)
+++ hive/trunk/itests/qtest/testconfiguration.properties Tue Jul  8 21:48:41 2014
@@ -1,5 +1,5 @@
 minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q
 minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q
 minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q
-minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q
+minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q
 beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q,
 exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwr
 ite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Jul  8 21:48:41 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -35,6 +36,7 @@ import org.apache.hadoop.hive.ql.QueryPl
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -365,6 +367,10 @@ public abstract class Task<T extends Ser
     return work;
   }
 
+  public Collection<MapWork> getMapWork() {
+    return Collections.<MapWork>emptyList();
+  }
+
   public void setId(String id) {
     this.id = id;
   }
@@ -389,7 +395,7 @@ public abstract class Task<T extends Ser
     return false;
   }
 
-  public Operator<? extends OperatorDesc> getReducer() {
+  public Operator<? extends OperatorDesc> getReducer(MapWork work) {
     return null;
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue Jul  8 21:48:41
2014
@@ -267,6 +267,10 @@ public final class Utilities {
     return w;
   }
 
+  public static void cacheMapWork(Configuration conf, MapWork work, Path hiveScratchDir)
{
+    cacheBaseWork(conf, MAP_PLAN_NAME, work, hiveScratchDir);
+  }
+
   public static void setMapWork(Configuration conf, MapWork work) {
     setBaseWork(conf, MAP_PLAN_NAME, work);
   }
@@ -283,6 +287,17 @@ public final class Utilities {
     return (ReduceWork) getBaseWork(conf, REDUCE_PLAN_NAME);
   }
 
+  public static void cacheBaseWork(Configuration conf, String name, BaseWork work,
+      Path hiveScratchDir) {
+    try {
+      setPlanPath(conf, hiveScratchDir);
+      setBaseWork(conf, name, work);
+    } catch (IOException e) {
+      LOG.error("Failed to cache plan", e);
+      throw new RuntimeException(e);
+    }
+  }
+
   /**
    * Pushes work into the global work map
    */
@@ -2324,13 +2339,15 @@ public final class Utilities {
 
   public static boolean isEmptyPath(JobConf job, Path dirPath, Context ctx)
       throws Exception {
-    ContentSummary cs = ctx.getCS(dirPath);
-    if (cs != null) {
-      LOG.info("Content Summary " + dirPath + "length: " + cs.getLength() + " num files:
"
-          + cs.getFileCount() + " num directories: " + cs.getDirectoryCount());
-      return (cs.getLength() == 0 && cs.getFileCount() == 0 && cs.getDirectoryCount()
<= 1);
-    } else {
-      LOG.info("Content Summary not cached for " + dirPath);
+    if (ctx != null) {
+      ContentSummary cs = ctx.getCS(dirPath);
+      if (cs != null) {
+        LOG.info("Content Summary " + dirPath + "length: " + cs.getLength() + " num files:
"
+            + cs.getFileCount() + " num directories: " + cs.getDirectoryCount());
+        return (cs.getLength() == 0 && cs.getFileCount() == 0 && cs.getDirectoryCount()
<= 1);
+      } else {
+        LOG.info("Content Summary not cached for " + dirPath);
+      }
     }
     return isEmptyPath(job, dirPath);
   }
@@ -2958,7 +2975,13 @@ public final class Utilities {
    * so we don't want to depend on scratch dir and context.
    */
   public static List<Path> getInputPathsTez(JobConf job, MapWork work) throws Exception
{
-    List<Path> paths = getInputPaths(job, work, null, null);
+    String scratchDir = HiveConf.getVar(job, HiveConf.ConfVars.SCRATCHDIR);
+
+    // we usually don't want to create dummy files for tez, however the metadata only
+    // optimization relies on it.
+    List<Path> paths = getInputPaths(job, work, new Path(scratchDir), null,
+        !work.isUseOneNullRowInputFormat());
+
     return paths;
   }
 
@@ -2976,8 +2999,8 @@ public final class Utilities {
    * @return List of paths to process for the given MapWork
    * @throws Exception
    */
-  public static List<Path> getInputPaths(JobConf job, MapWork work, Path hiveScratchDir,
Context ctx)
-      throws Exception {
+  public static List<Path> getInputPaths(JobConf job, MapWork work, Path hiveScratchDir,
+      Context ctx, boolean skipDummy) throws Exception {
     int sequenceNumber = 0;
 
     Set<Path> pathsProcessed = new HashSet<Path>();
@@ -3002,7 +3025,7 @@ public final class Utilities {
           pathsProcessed.add(path);
 
           LOG.info("Adding input file " + path);
-          if (!HiveConf.getVar(job, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")
+          if (!skipDummy
               && isEmptyPath(job, path, ctx)) {
             path = createDummyFileForEmptyPartition(path, job, work,
                  hiveScratchDir, alias, sequenceNumber++);
@@ -3020,8 +3043,7 @@ public final class Utilities {
       // T2) x;
       // If T is empty and T2 contains 100 rows, the user expects: 0, 100 (2
       // rows)
-      if (path == null
-          && !HiveConf.getVar(job, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez"))
{
+      if (path == null && !skipDummy) {
         path = createDummyFileForEmptyTable(job, work, hiveScratchDir,
             alias, sequenceNumber++);
         pathsToAdd.add(path);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Tue Jul  8 21:48:41
2014
@@ -366,7 +366,7 @@ public class ExecDriver extends Task<Map
         }
       }
       work.configureJobConf(job);
-      List<Path> inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDir,
ctx);
+      List<Path> inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDir,
ctx, false);
       Utilities.setInputPaths(job, inputPaths);
 
       Utilities.setMapRedWork(job, work, ctx.getMRTmpPath());
@@ -789,6 +789,11 @@ public class ExecDriver extends Task<Map
   }
 
   @Override
+  public Collection<MapWork> getMapWork() {
+    return Collections.<MapWork>singleton(getWork().getMapWork());
+  }
+
+  @Override
   public boolean isMapRedTask() {
     return true;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java Tue Jul  8 21:48:41
2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.DriverC
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -462,8 +463,11 @@ public class MapRedTask extends ExecDriv
   }
 
   @Override
-  public Operator<? extends OperatorDesc> getReducer() {
-    return getWork().getReduceWork() == null ? null : getWork().getReduceWork().getReducer();
+  public Operator<? extends OperatorDesc> getReducer(MapWork mapWork) {
+    if (getWork().getMapWork() == mapWork) {
+      return getWork().getReduceWork() == null ? null : getWork().getReduceWork().getReducer();
+    }
+    return null;
   }
 
   @Override

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Tue Jul  8 21:48:41
2014
@@ -49,6 +49,7 @@ import org.apache.hadoop.hive.ql.exec.mr
 import org.apache.hadoop.hive.ql.exec.mr.ExecReducer;
 import org.apache.hadoop.hive.ql.exec.tez.tools.TezMergedLogicalInput;
 import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
 import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.io.HiveKey;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
@@ -195,6 +196,10 @@ public class DagUtils {
       inpFormat = BucketizedHiveInputFormat.class.getName();
     }
 
+    if (mapWork.isUseOneNullRowInputFormat()) {
+      inpFormat = CombineHiveInputFormat.class.getName();
+    }
+
     conf.set("mapred.mapper.class", ExecMapper.class.getName());
     conf.set("mapred.input.format.class", inpFormat);
 
@@ -413,7 +418,7 @@ public class DagUtils {
     Path tezDir = getTezDir(mrScratchDir);
 
     // set up the operator plan
-    Utilities.setMapWork(conf, mapWork, mrScratchDir, false);
+    Utilities.cacheMapWork(conf, mapWork, mrScratchDir);
 
     // create the directories FileSinkOperators need
     Utilities.createTmpDirs(conf, mapWork);
@@ -441,6 +446,7 @@ public class DagUtils {
         }
       }
     }
+
     if (vertexHasCustomInput) {
       useTezGroupedSplits = false;
       // grouping happens in execution phase. Setting the class to TezGroupedSplitsInputFormat
@@ -459,7 +465,8 @@ public class DagUtils {
       }
     }
 
-    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_AM_SPLIT_GENERATION)) {
+    if (HiveConf.getBoolVar(conf, ConfVars.HIVE_AM_SPLIT_GENERATION)
+        && !mapWork.isUseOneNullRowInputFormat()) {
       // if we're generating the splits in the AM, we just need to set
       // the correct plugin.
       amSplitGeneratorClass = HiveSplitGenerator.class;
@@ -470,6 +477,9 @@ public class DagUtils {
       numTasks = inputSplitInfo.getNumTasks();
     }
 
+    // set up the operator plan
+    Utilities.setMapWork(conf, mapWork, mrScratchDir, false);
+
     byte[] serializedConf = MRHelpers.createUserPayloadFromConf(conf);
     map = new Vertex(mapWork.getName(),
         new ProcessorDescriptor(MapTezProcessor.class.getName()).

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Tue Jul  8 21:48:41
2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.exec.tez;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -36,6 +37,9 @@ import org.apache.hadoop.hive.ql.exec.Ta
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.TezEdgeProperty;
 import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType;
 import org.apache.hadoop.hive.ql.plan.TezWork;
@@ -354,4 +358,42 @@ public class TezTask extends Task<TezWor
   public String getName() {
     return "TEZ";
   }
+
+  @Override
+  public Collection<MapWork> getMapWork() {
+    List<MapWork> result = new LinkedList<MapWork>();
+    TezWork work = getWork();
+
+    // framework expects MapWork instances that have no physical parents (i.e.: union parent
is
+    // fine, broadcast parent isn't)
+    for (BaseWork w: work.getAllWorkUnsorted()) {
+      if (w instanceof MapWork) {
+        List<BaseWork> parents = work.getParents(w);
+        boolean candidate = true;
+        for (BaseWork parent: parents) {
+          if (!(parent instanceof UnionWork)) {
+            candidate = false;
+          }
+        }
+        if (candidate) {
+          result.add((MapWork)w);
+        }
+      }
+    }
+    return result;
+  }
+
+  @Override
+  public Operator<? extends OperatorDesc> getReducer(MapWork mapWork) {
+    List<BaseWork> children = getWork().getChildren(mapWork);
+    if (children.size() != 1) {
+      return null;
+    }
+
+    if (!(children.get(0) instanceof ReduceWork)) {
+      return null;
+    }
+
+    return ((ReduceWork)children.get(0)).getReducer();
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
Tue Jul  8 21:48:41 2014
@@ -230,6 +230,8 @@ public class MetadataOnlyOptimizer imple
     }
 
     private void processAlias(MapWork work, String alias) {
+      work.setUseOneNullRowInputFormat(true);
+
       // Change the alias partition desc
       PartitionDesc aliasPartn = work.getAliasToPartnInfo().get(alias);
       changePartitionToMetadataOnly(aliasPartn);
@@ -258,13 +260,6 @@ public class MetadataOnlyOptimizer imple
         throws SemanticException {
       Task<? extends Serializable> task = (Task<? extends Serializable>) nd;
 
-      Collection<Operator<? extends OperatorDesc>> topOperators
-        = task.getTopOperators();
-      if (topOperators.size() == 0) {
-        return null;
-      }
-
-      LOG.info("Looking for table scans where optimization is applicable");
       // create a the context for walking operators
       ParseContext parseContext = physicalContext.getParseContext();
       WalkerCtx walkerCtx = new WalkerCtx();
@@ -277,41 +272,52 @@ public class MetadataOnlyOptimizer imple
         GroupByOperator.getOperatorName() + "%.*" + FileSinkOperator.getOperatorName() +
"%"),
         new FileSinkProcessor());
 
-      // The dispatcher fires the processor corresponding to the closest
-      // matching rule and passes the context along
-      Dispatcher disp = new DefaultRuleDispatcher(null, opRules, walkerCtx);
-      GraphWalker ogw = new PreOrderWalker(disp);
-
-      // Create a list of topOp nodes
-      ArrayList<Node> topNodes = new ArrayList<Node>();
-      // Get the top Nodes for this map-reduce task
-      for (Operator<? extends OperatorDesc>
-           workOperator : topOperators) {
-        if (parseContext.getTopOps().values().contains(workOperator)) {
-          topNodes.add(workOperator);
+      for (MapWork mapWork: task.getMapWork()) {
+        LOG.debug("Looking at: "+mapWork.getName());
+        Collection<Operator<? extends OperatorDesc>> topOperators
+          = mapWork.getAliasToWork().values();
+        if (topOperators.size() == 0) {
+          LOG.debug("No top operators");
+          return null;
         }
-      }
 
-      if (task.getReducer() != null) {
-        topNodes.add(task.getReducer());
-      }
+        LOG.info("Looking for table scans where optimization is applicable");
 
-      ogw.startWalking(topNodes, null);
+        // The dispatcher fires the processor corresponding to the closest
+        // matching rule and passes the context along
+        Dispatcher disp = new DefaultRuleDispatcher(null, opRules, walkerCtx);
+        GraphWalker ogw = new PreOrderWalker(disp);
+
+        // Create a list of topOp nodes
+        ArrayList<Node> topNodes = new ArrayList<Node>();
+        // Get the top Nodes for this map-reduce task
+        for (Operator<? extends OperatorDesc>
+               workOperator : topOperators) {
+          if (parseContext.getTopOps().values().contains(workOperator)) {
+            topNodes.add(workOperator);
+          }
+        }
 
-      LOG.info(String.format("Found %d metadata only table scans",
-          walkerCtx.getMetadataOnlyTableScans().size()));
-      Iterator<TableScanOperator> iterator
-        = walkerCtx.getMetadataOnlyTableScans().iterator();
-
-      while (iterator.hasNext()) {
-        TableScanOperator tso = iterator.next();
-        ((TableScanDesc)tso.getConf()).setIsMetadataOnly(true);
-        MapWork work = ((MapredWork) task.getWork()).getMapWork();
-        String alias = getAliasForTableScanOperator(work, tso);
-        LOG.info("Metadata only table scan for " + alias);
-        processAlias(work, alias);
-      }
+        Operator<? extends OperatorDesc> reducer = task.getReducer(mapWork);
+        if (reducer != null) {
+          topNodes.add(reducer);
+        }
+
+        ogw.startWalking(topNodes, null);
 
+        LOG.info(String.format("Found %d metadata only table scans",
+            walkerCtx.getMetadataOnlyTableScans().size()));
+        Iterator<TableScanOperator> iterator
+          = walkerCtx.getMetadataOnlyTableScans().iterator();
+
+        while (iterator.hasNext()) {
+          TableScanOperator tso = iterator.next();
+          ((TableScanDesc)tso.getConf()).setIsMetadataOnly(true);
+          String alias = getAliasForTableScanOperator(mapWork, tso);
+          LOG.info("Metadata only table scan for " + alias);
+          processAlias(mapWork, alias);
+        }
+      }
       return null;
     }
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java Tue Jul  8 21:48:41
2014
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.optimiz
 import org.apache.hadoop.hive.ql.optimizer.ReduceSinkMapJoinProc;
 import org.apache.hadoop.hive.ql.optimizer.SetReducerParallelism;
 import org.apache.hadoop.hive.ql.optimizer.physical.CrossProductCheck;
+import org.apache.hadoop.hive.ql.optimizer.physical.MetadataOnlyOptimizer;
 import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
 import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
@@ -247,15 +248,28 @@ public class TezCompiler extends TaskCom
     PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks,
        pCtx.getFetchTask());
 
+    if (conf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
+      physicalCtx = new MetadataOnlyOptimizer().resolve(physicalCtx);
+    } else {
+      LOG.debug("Skipping metadata only query optimization");
+    }
+
     if (conf.getBoolVar(HiveConf.ConfVars.HIVE_CHECK_CROSS_PRODUCT)) {
       physicalCtx = new CrossProductCheck().resolve(physicalCtx);
+    } else {
+      LOG.debug("Skipping cross product analysis");
     }
 
     if (conf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
-      (new Vectorizer()).resolve(physicalCtx);
+      physicalCtx = new Vectorizer().resolve(physicalCtx);
+    } else {
+      LOG.debug("Skipping vectorization");
     }
+
     if (!"none".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE))) {
-      (new StageIDsRearranger()).resolve(physicalCtx);
+      physicalCtx = new StageIDsRearranger().resolve(physicalCtx);
+    } else {
+      LOG.debug("Skipping stage id rearranger");
     }
     return;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java Tue Jul  8 21:48:41
2014
@@ -114,6 +114,8 @@ public class MapWork extends BaseWork {
 
   private boolean useBucketizedHiveInputFormat;
 
+  private boolean useOneNullRowInputFormat;
+
   private Map<String, Map<Integer, String>> scratchColumnVectorTypes = null;
   private Map<String, Map<String, Integer>> scratchColumnMap = null;
   private boolean vectorMode = false;
@@ -390,6 +392,7 @@ public class MapWork extends BaseWork {
   public void setInputformat(String inputformat) {
     this.inputformat = inputformat;
   }
+
   public boolean isUseBucketizedHiveInputFormat() {
     return useBucketizedHiveInputFormat;
   }
@@ -398,6 +401,14 @@ public class MapWork extends BaseWork {
     this.useBucketizedHiveInputFormat = useBucketizedHiveInputFormat;
   }
 
+  public void setUseOneNullRowInputFormat(boolean useOneNullRowInputFormat) {
+    this.useOneNullRowInputFormat = useOneNullRowInputFormat;
+  }
+
+  public boolean isUseOneNullRowInputFormat() {
+    return useOneNullRowInputFormat;
+  }
+
   public QBJoinTree getJoinTree() {
     return joinTree;
   }
@@ -534,4 +545,14 @@ public class MapWork extends BaseWork {
     this.vectorMode = vectorMode;
   }
 
+  public void logPathToAliases() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("LOGGING PATH TO ALIASES");
+      for (Map.Entry<String, ArrayList<String>> entry: pathToAliases.entrySet())
{
+        for (String a: entry.getValue()) {
+          LOG.debug("Path: " + entry.getKey() + ", Alias: " + a);
+        }
+      }
+    }
+  }
 }

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java?rev=1608959&r1=1608958&r2=1608959&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestSymlinkTextInputFormat.java Tue
Jul  8 21:48:41 2014
@@ -177,7 +177,7 @@ public class TestSymlinkTextInputFormat 
       QueryPlan plan = drv.getPlan();
       MapRedTask selectTask = (MapRedTask)plan.getRootTasks().get(0);
 
-      List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(),
emptyScratchDir, ctx);
+      List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(),
emptyScratchDir, ctx, false);
       Utilities.setInputPaths(newJob, inputPaths);
 
       Utilities.setMapRedWork(newJob, selectTask.getWork(), ctx.getMRTmpPath());

Added: hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries_with_filters.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries_with_filters.q.out?rev=1608959&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries_with_filters.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/metadata_only_queries_with_filters.q.out
Tue Jul  8 21:48:41 2014
@@ -0,0 +1,196 @@
+PREHOOK: query: create table over10k(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp, 
+           dec decimal,  
+           bin binary)
+       row format delimited
+       fields terminated by '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table over10k(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp, 
+           dec decimal,  
+           bin binary)
+       row format delimited
+       fields terminated by '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over10k
+PREHOOK: query: load data local inpath '../../data/files/over10k' into table over10k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@over10k
+POSTHOOK: query: load data local inpath '../../data/files/over10k' into table over10k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@over10k
+PREHOOK: query: create table stats_tbl_part(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp, 
+           dec decimal,  
+           bin binary) partitioned by (dt int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table stats_tbl_part(
+           t tinyint,
+           si smallint,
+           i int,
+           b bigint,
+           f float,
+           d double,
+           bo boolean,
+           s string,
+           ts timestamp, 
+           dec decimal,  
+           bin binary) partitioned by (dt int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@stats_tbl_part
+PREHOOK: query: from over10k 
+insert overwrite table stats_tbl_part partition (dt=2010) select t,si,i,b,f,d,bo,s,ts,dec,bin
where t>0 and t<30 
+insert overwrite table stats_tbl_part partition (dt=2014) select t,si,i,b,f,d,bo,s,ts,dec,bin
where t > 30 and t<60
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+PREHOOK: Output: default@stats_tbl_part@dt=2010
+PREHOOK: Output: default@stats_tbl_part@dt=2014
+POSTHOOK: query: from over10k 
+insert overwrite table stats_tbl_part partition (dt=2010) select t,si,i,b,f,d,bo,s,ts,dec,bin
where t>0 and t<30 
+insert overwrite table stats_tbl_part partition (dt=2014) select t,si,i,b,f,d,bo,s,ts,dec,bin
where t > 30 and t<60
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+POSTHOOK: Output: default@stats_tbl_part@dt=2010
+POSTHOOK: Output: default@stats_tbl_part@dt=2014
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).b SIMPLE [(over10k)over10k.FieldSchema(name:b,
type:bigint, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).bin SIMPLE [(over10k)over10k.FieldSchema(name:bin,
type:binary, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).bo SIMPLE [(over10k)over10k.FieldSchema(name:bo,
type:boolean, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).d SIMPLE [(over10k)over10k.FieldSchema(name:d,
type:double, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).dec SIMPLE [(over10k)over10k.FieldSchema(name:dec,
type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).f SIMPLE [(over10k)over10k.FieldSchema(name:f,
type:float, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).i SIMPLE [(over10k)over10k.FieldSchema(name:i,
type:int, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).s SIMPLE [(over10k)over10k.FieldSchema(name:s,
type:string, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).si SIMPLE [(over10k)over10k.FieldSchema(name:si,
type:smallint, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).t SIMPLE [(over10k)over10k.FieldSchema(name:t,
type:tinyint, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2010).ts SIMPLE [(over10k)over10k.FieldSchema(name:ts,
type:timestamp, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).b SIMPLE [(over10k)over10k.FieldSchema(name:b,
type:bigint, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).bin SIMPLE [(over10k)over10k.FieldSchema(name:bin,
type:binary, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).bo SIMPLE [(over10k)over10k.FieldSchema(name:bo,
type:boolean, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).d SIMPLE [(over10k)over10k.FieldSchema(name:d,
type:double, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).dec SIMPLE [(over10k)over10k.FieldSchema(name:dec,
type:decimal(10,0), comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).f SIMPLE [(over10k)over10k.FieldSchema(name:f,
type:float, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).i SIMPLE [(over10k)over10k.FieldSchema(name:i,
type:int, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).s SIMPLE [(over10k)over10k.FieldSchema(name:s,
type:string, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).si SIMPLE [(over10k)over10k.FieldSchema(name:si,
type:smallint, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).t SIMPLE [(over10k)over10k.FieldSchema(name:t,
type:tinyint, comment:null), ]
+POSTHOOK: Lineage: stats_tbl_part PARTITION(dt=2014).ts SIMPLE [(over10k)over10k.FieldSchema(name:ts,
type:timestamp, comment:null), ]
+PREHOOK: query: analyze table stats_tbl_part partition(dt) compute statistics
+PREHOOK: type: QUERY
+PREHOOK: Input: default@stats_tbl_part
+PREHOOK: Input: default@stats_tbl_part@dt=2010
+PREHOOK: Input: default@stats_tbl_part@dt=2014
+PREHOOK: Output: default@stats_tbl_part
+PREHOOK: Output: default@stats_tbl_part@dt=2010
+PREHOOK: Output: default@stats_tbl_part@dt=2014
+POSTHOOK: query: analyze table stats_tbl_part partition(dt) compute statistics
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@stats_tbl_part
+POSTHOOK: Input: default@stats_tbl_part@dt=2010
+POSTHOOK: Input: default@stats_tbl_part@dt=2014
+POSTHOOK: Output: default@stats_tbl_part
+POSTHOOK: Output: default@stats_tbl_part@dt=2010
+POSTHOOK: Output: default@stats_tbl_part@dt=2014
+PREHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns
t,si,i,b,f,d,bo,s,bin
+PREHOOK: type: QUERY
+PREHOOK: Input: default@stats_tbl_part
+PREHOOK: Input: default@stats_tbl_part@dt=2010
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_tbl_part partition(dt=2010) compute statistics for columns
t,si,i,b,f,d,bo,s,bin
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@stats_tbl_part
+POSTHOOK: Input: default@stats_tbl_part@dt=2010
+#### A masked pattern was here ####
+PREHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns
t,si,i,b,f,d,bo,s,bin
+PREHOOK: type: QUERY
+PREHOOK: Input: default@stats_tbl_part
+PREHOOK: Input: default@stats_tbl_part@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: analyze table stats_tbl_part partition(dt=2014) compute statistics for columns
t,si,i,b,f,d,bo,s,bin
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@stats_tbl_part
+POSTHOOK: Input: default@stats_tbl_part@dt=2014
+#### A masked pattern was here ####
+PREHOOK: query: explain 
+select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b),
max(f), min(d) from stats_tbl_part where dt = 2010
+PREHOOK: type: QUERY
+POSTHOOK: query: explain 
+select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si), max(i), min(b),
max(f), min(d) from stats_tbl_part where dt = 2010
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: 1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si),
max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010
+PREHOOK: type: QUERY
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*), count(1), sum(1), count(s), count(bo), count(bin), count(si),
max(i), min(b), max(f), min(d) from stats_tbl_part where dt = 2010
+POSTHOOK: type: QUERY
+#### A masked pattern was here ####
+2322	2322	2322	2322	2322	2322	2322	65791	4294967296	99.9800033569336	0.03
+PREHOOK: query: explain 
+select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin), count(si), max(i),
min(b), max(f), min(d) from stats_tbl_part where dt > 2010
+PREHOOK: type: QUERY
+POSTHOOK: query: explain 
+select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin), count(si), max(i),
min(b), max(f), min(d) from stats_tbl_part where dt > 2010
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: 1
+      Processor Tree:
+        ListSink
+
+PREHOOK: query: select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin),
count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt > 2010
+PREHOOK: type: QUERY
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin),
count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt > 2010
+POSTHOOK: type: QUERY
+#### A masked pattern was here ####
+2219	2219	2219	4438	2219	2219	2219	2219	65791	4294967296	99.95999908447266	0.04
+PREHOOK: query: drop table stats_tbl_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@stats_tbl_part
+PREHOOK: Output: default@stats_tbl_part
+POSTHOOK: query: drop table stats_tbl_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@stats_tbl_part
+POSTHOOK: Output: default@stats_tbl_part

Added: hive/trunk/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/metadataonly1.q.out?rev=1608959&view=auto
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/metadataonly1.q.out (added) and hive/trunk/ql/src/test/results/clientpositive/tez/metadataonly1.q.out
Tue Jul  8 21:48:41 2014 differ



Mime
View raw message