hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1580169 [1/2] - in /hive/branches/branch-0.13/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/ java/org/apache/hadoop/hive/ql/...
Date Sat, 22 Mar 2014 05:03:42 GMT
Author: hashutosh
Date: Sat Mar 22 05:03:41 2014
New Revision: 1580169

URL: http://svn.apache.org/r1580169
Log:
HIVE-4293 : Predicates following UDTF operator are removed by PPD (Navis via Harish Butani)

Added:
    hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_join4.q
    hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_udtf.q
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_join4.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_udtf.q.out
Modified:
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java
    hive/branches/branch-0.13/ql/src/test/queries/clientpositive/lateral_view_ppd.q
    hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_transform.q
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/cluster.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ctas_colname.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd2.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby2.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_transform.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/subquery_notin_having.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
    hive/branches/branch-0.13/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join1.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join2.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join3.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join4.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join5.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join6.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join7.q.xml
    hive/branches/branch-0.13/ql/src/test/results/compiler/plan/join8.q.xml

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java Sat Mar 22 05:03:41 2014
@@ -78,8 +78,8 @@ public class LateralViewJoinOperator ext
 
   // The expected tags from the parent operators. See processOp() before
   // changing the tags.
-  public static final int SELECT_TAG = 0;
-  public static final int UDTF_TAG = 1;
+  public static final byte SELECT_TAG = 0;
+  public static final byte UDTF_TAG = 1;
 
   @Override
   protected void initializeOp(Configuration hconf) throws HiveException {

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Sat Mar 22 05:03:41 2014
@@ -55,6 +55,10 @@ import org.apache.hadoop.mapred.OutputCo
 public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
     implements Serializable, TopNHash.BinaryCollector {
 
+  static {
+    PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases");
+  }
+
   private static final long serialVersionUID = 1L;
   protected transient OutputCollector out;
 
@@ -84,14 +88,14 @@ public class ReduceSinkOperator extends 
   protected transient byte[] tagByte = new byte[1];
   transient protected int numDistributionKeys;
   transient protected int numDistinctExprs;
-  transient String inputAlias;  // input alias of this RS for join (used for PPD)
+  transient String[] inputAliases;  // input aliases of this RS for join (used for PPD)
 
-  public void setInputAlias(String inputAlias) {
-    this.inputAlias = inputAlias;
+  public void setInputAliases(String[] inputAliases) {
+    this.inputAliases = inputAliases;
   }
 
-  public String getInputAlias() {
-    return inputAlias;
+  public String[] getInputAliases() {
+    return inputAliases;
   }
 
   public void setOutputCollector(OutputCollector _out) {

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Sat Mar 22 05:03:41 2014
@@ -474,39 +474,38 @@ public final class ColumnPrunerProcFacto
       List<String> cols = cppCtx.genColLists(op);
 
       Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap();
-
       // As columns go down the DAG, the LVJ will transform internal column
       // names from something like 'key' to '_col0'. Because of this, we need
       // to undo this transformation using the column expression map as the
       // column names propagate up the DAG.
-      List<String> colsAfterReplacement = new ArrayList<String>();
-      for (String col : cols) {
-        if (colExprMap.containsKey(col)) {
-          ExprNodeDesc expr = colExprMap.get(col);
-          colsAfterReplacement.addAll(expr.getCols());
-        } else {
-          colsAfterReplacement.add(col);
-        }
-      }
+
       // this is SEL(*) cols + UDTF cols
       List<String> outputCols = op.getConf().getOutputInternalColNames();
-      if (outputCols.size() != cols.size()) {
-        // cause we cannot prune columns from UDTF branch currently, extract
-        // columns from SEL(*) branch only and append all columns from UDTF branch to it
-        ArrayList<String> newColNames = new ArrayList<String>();
-        for (String col : cols) {
-          int index = outputCols.indexOf(col);
-          // colExprMap.size() == size of cols from SEL(*) branch
-          if (index >= 0 && index < colExprMap.size()) {
-            newColNames.add(col);
-          }
-        }
-        newColNames.addAll(outputCols.subList(colExprMap.size(), outputCols.size()));
-        op.getConf().setOutputInternalColNames(newColNames);
-      }
 
-      cppCtx.getPrunedColLists().put(op,
-          colsAfterReplacement);
+      // cause we cannot prune columns from UDTF branch currently, extract
+      // columns from SEL(*) branch only and append all columns from UDTF branch to it
+      int numSelColumns = op.getConf().getNumSelColumns();
+
+      List<String> colsAfterReplacement = new ArrayList<String>();
+      ArrayList<String> newColNames = new ArrayList<String>();
+      for (String col : cols) {
+        int index = outputCols.indexOf(col);
+        // colExprMap.size() == size of cols from SEL(*) branch
+        if (index >= 0 && index < numSelColumns) {
+          ExprNodeDesc transformed = colExprMap.get(col);
+          Utilities.mergeUniqElems(colsAfterReplacement, transformed.getCols());
+          newColNames.add(col);
+        }
+      }
+      // update number of columns from sel(*)
+      op.getConf().setNumSelColumns(newColNames.size());
+
+      // add all UDTF columns
+      // following SEL will do CP for columns from UDTF, not adding SEL in here
+      newColNames.addAll(outputCols.subList(numSelColumns, outputCols.size()));
+      op.getConf().setOutputInternalColNames(newColNames);
+
+      cppCtx.getPrunedColLists().put(op, colsAfterReplacement);
       return null;
     }
   }

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Mar 22 05:03:41 2014
@@ -6587,7 +6587,7 @@ public class SemanticAnalyzer extends Ba
 
   @SuppressWarnings("nls")
   private Operator genJoinReduceSinkChild(QB qb, QBJoinTree joinTree,
-      Operator child, String srcName, int pos) throws SemanticException {
+      Operator child, String[] srcs, int pos) throws SemanticException {
     RowResolver inputRS = opParseCtx.get(child).getRowResolver();
     RowResolver outputRS = new RowResolver();
     ArrayList<String> outputColumns = new ArrayList<String>();
@@ -6647,7 +6647,7 @@ public class SemanticAnalyzer extends Ba
             reduceKeys.size(), numReds), new RowSchema(outputRS
             .getColumnInfos()), child), outputRS);
     rsOp.setColumnExprMap(colExprMap);
-    rsOp.setInputAlias(srcName);
+    rsOp.setInputAliases(srcs);
     return rsOp;
   }
 
@@ -6666,7 +6666,8 @@ public class SemanticAnalyzer extends Ba
       for (ASTNode cond : filter) {
         joinSrcOp = genFilterPlan(qb, cond, joinSrcOp);
       }
-      joinSrcOp = genJoinReduceSinkChild(qb, joinTree, joinSrcOp, null, 0);
+      String[] leftAliases = joinTree.getLeftAliases();
+      joinSrcOp = genJoinReduceSinkChild(qb, joinTree, joinSrcOp, leftAliases, 0);
     }
 
     Operator[] srcOps = new Operator[joinTree.getBaseSrc().length];
@@ -6698,7 +6699,7 @@ public class SemanticAnalyzer extends Ba
         }
 
         // generate a ReduceSink operator for the join
-        srcOps[pos] = genJoinReduceSinkChild(qb, joinTree, srcOp, src, pos);
+        srcOps[pos] = genJoinReduceSinkChild(qb, joinTree, srcOp, new String[]{src}, pos);
         pos++;
       } else {
         assert pos == 0;
@@ -9073,6 +9074,7 @@ public class SemanticAnalyzer extends Ba
     Operator allPath = putOpInsertMap(OperatorFactory.getAndMakeChild(
         new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()),
         lvForward), allPathRR);
+    int allColumns = allPathRR.getColumnInfos().size();
     // Get the UDTF Path
     QB blankQb = new QB(null, null, false);
     Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
@@ -9092,8 +9094,6 @@ public class SemanticAnalyzer extends Ba
     RowResolver lateralViewRR = new RowResolver();
     ArrayList<String> outputInternalColNames = new ArrayList<String>();
 
-    LVmergeRowResolvers(allPathRR, lateralViewRR, outputInternalColNames);
-    LVmergeRowResolvers(udtfPathRR, lateralViewRR, outputInternalColNames);
 
     // For PPD, we need a column to expression map so that during the walk,
     // the processor knows how to transform the internal col names.
@@ -9101,17 +9101,11 @@ public class SemanticAnalyzer extends Ba
     // LVmerge.. in the above order
     Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
 
-    int i = 0;
-    for (ColumnInfo c : allPathRR.getColumnInfos()) {
-      String internalName = getColumnInternalName(i);
-      i++;
-      colExprMap.put(internalName,
-          new ExprNodeColumnDesc(c.getType(), c.getInternalName(),
-              c.getTabAlias(), c.getIsVirtualCol()));
-    }
+    LVmergeRowResolvers(allPathRR, lateralViewRR, colExprMap, outputInternalColNames);
+    LVmergeRowResolvers(udtfPathRR, lateralViewRR, colExprMap, outputInternalColNames);
 
     Operator lateralViewJoin = putOpInsertMap(OperatorFactory
-        .getAndMakeChild(new LateralViewJoinDesc(outputInternalColNames),
+        .getAndMakeChild(new LateralViewJoinDesc(allColumns, outputInternalColNames),
             new RowSchema(lateralViewRR.getColumnInfos()), allPath,
             udtfPath), lateralViewRR);
     lateralViewJoin.setColumnExprMap(colExprMap);
@@ -9134,7 +9128,7 @@ public class SemanticAnalyzer extends Ba
    *          the same order as in the dest row resolver
    */
   private void LVmergeRowResolvers(RowResolver source, RowResolver dest,
-      ArrayList<String> outputInternalColNames) {
+      Map<String, ExprNodeDesc> colExprMap, ArrayList<String> outputInternalColNames) {
     for (ColumnInfo c : source.getColumnInfos()) {
       String internalName = getColumnInternalName(outputInternalColNames.size());
       outputInternalColNames.add(internalName);
@@ -9144,6 +9138,8 @@ public class SemanticAnalyzer extends Ba
       String tableAlias = tableCol[0];
       String colAlias = tableCol[1];
       dest.put(tableAlias, colAlias, newCol);
+      colExprMap.put(internalName, new ExprNodeColumnDesc(c.getType(), c.getInternalName(),
+          c.getTabAlias(), c.getIsVirtualCol()));
     }
   }
 

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java Sat Mar 22 05:03:41 2014
@@ -29,12 +29,14 @@ import java.util.ArrayList;
 public class LateralViewJoinDesc extends AbstractOperatorDesc {
   private static final long serialVersionUID = 1L;
 
+  private int numSelColumns;
   private ArrayList<String> outputInternalColNames;
 
   public LateralViewJoinDesc() {
   }
 
-  public LateralViewJoinDesc(ArrayList<String> outputInternalColNames) {
+  public LateralViewJoinDesc(int numSelColumns, ArrayList<String> outputInternalColNames) {
+    this.numSelColumns = numSelColumns;
     this.outputInternalColNames = outputInternalColNames;
   }
 
@@ -46,4 +48,12 @@ public class LateralViewJoinDesc extends
   public ArrayList<String> getOutputInternalColNames() {
     return outputInternalColNames;
   }
+
+  public int getNumSelColumns() {
+    return numSelColumns;
+  }
+
+  public void setNumSelColumns(int numSelColumns) {
+    this.numSelColumns = numSelColumns;
+  }
 }

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java Sat Mar 22 05:03:41 2014
@@ -19,9 +19,11 @@ package org.apache.hadoop.hive.ql.ppd;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -289,6 +291,21 @@ public class ExprWalkerInfo implements N
     return nonFinalPreds;
   }
 
+  public Map<String, List<ExprNodeDesc>> getResidualPredicates(boolean clear) {
+    Map<String, List<ExprNodeDesc>> oldExprs = new HashMap<String, List<ExprNodeDesc>>();
+    for (Map.Entry<String, List<ExprNodeDesc>> entry : nonFinalPreds.entrySet()) {
+      List<ExprNodeDesc> converted = new ArrayList<ExprNodeDesc>();
+      for (ExprNodeDesc newExpr : entry.getValue()) {
+        converted.add(newToOldExprMap.get(newExpr));
+      }
+      oldExprs.put(entry.getKey(), converted);
+    }
+    if (clear) {
+      nonFinalPreds.clear();
+    }
+    return oldExprs;
+  }
+
   /**
    * Merges the specified pushdown predicates with the current class.
    *

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Sat Mar 22 05:03:41 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
 import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
@@ -81,6 +82,32 @@ public final class OpProcFactory {
   protected static final Log LOG = LogFactory.getLog(OpProcFactory.class
     .getName());
 
+  private static ExprWalkerInfo getChildWalkerInfo(Operator<?> current, OpWalkerInfo owi) {
+    if (current.getNumChild() == 0) {
+      return null;
+    }
+    if (current.getNumChild() > 1) {
+      // ppd for multi-insert query is not yet implemented
+      // no-op for leafs
+      for (Operator<?> child : current.getChildOperators()) {
+        removeCandidates(child, owi); // remove candidated filters on this branch
+      }
+      return null;
+    }
+    return owi.getPrunedPreds(current.getChildOperators().get(0));
+  }
+
+  private static void removeCandidates(Operator<?> operator, OpWalkerInfo owi) {
+    if (operator instanceof FilterOperator) {
+      owi.getCandidateFilterOps().remove(operator);
+    }
+    if (operator.getChildOperators() != null) {
+      for (Operator<?> child : operator.getChildOperators()) {
+        removeCandidates(child, owi);
+      }
+    }
+  }
+
   /**
    * Processor for Script Operator Prevents any predicates being pushed.
    */
@@ -96,7 +123,8 @@ public final class OpProcFactory {
       // same with LIMIT op
       // create a filter with all children predicates
       OpWalkerInfo owi = (OpWalkerInfo) procCtx;
-      if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+      ExprWalkerInfo childInfo = getChildWalkerInfo((Operator<?>) nd, owi);
+      if (childInfo != null && HiveConf.getBoolVar(owi.getParseContext().getConf(),
           HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
         ExprWalkerInfo unpushedPreds = mergeChildrenPred(nd, owi, null, false);
         return createFilter((Operator)nd, unpushedPreds, owi);
@@ -110,10 +138,17 @@ public final class OpProcFactory {
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
-      LOG.info("Processing for " + nd.getName() + "("
-          + ((Operator) nd).getIdentifier() + ")");
-      //Predicates for UDTF wont be candidates for its children. So, nothing to
-      //optimize here. See lateral_view_ppd.q for example.
+      super.process(nd, stack, procCtx, nodeOutputs);
+      OpWalkerInfo owi = (OpWalkerInfo) procCtx;
+      ExprWalkerInfo prunedPred = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd);
+      if (prunedPred == null) {
+        return null;
+      }
+      Map<String, List<ExprNodeDesc>> candidates = prunedPred.getFinalCandidates();
+      if (candidates != null && !candidates.isEmpty()) {
+        createFilter((Operator)nd, prunedPred, owi);
+        candidates.clear();
+      }
       return null;
     }
 
@@ -155,7 +190,7 @@ public final class OpProcFactory {
           + ((Operator) nd).getIdentifier() + ")");
       OpWalkerInfo owi = (OpWalkerInfo) procCtx;
       TableScanOperator tsOp = (TableScanOperator) nd;
-      mergeWithChildrenPred(tsOp, owi, null, null, false);
+      mergeWithChildrenPred(tsOp, owi, null, null);
       ExprWalkerInfo pushDownPreds = owi.getPrunedPreds(tsOp);
       return createFilter(tsOp, pushDownPreds, owi);
     }
@@ -204,7 +239,7 @@ public final class OpProcFactory {
         owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, ewi);
       }
       // merge it with children predicates
-      boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, ewi, null, false);
+      boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, ewi, null);
       if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
           HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
         if (hasUnpushedPredicates) {
@@ -220,20 +255,18 @@ public final class OpProcFactory {
    * Determines predicates for which alias can be pushed to it's parents. See
    * the comments for getQualifiedAliases function.
    */
-  public static class JoinPPD extends DefaultPPD implements NodeProcessor {
+  public static class JoinerPPD extends DefaultPPD implements NodeProcessor {
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
       LOG.info("Processing for " + nd.getName() + "("
           + ((Operator) nd).getIdentifier() + ")");
       OpWalkerInfo owi = (OpWalkerInfo) procCtx;
-      Set<String> aliases = getQualifiedAliases((JoinOperator) nd, owi
-          .getRowResolver(nd));
+      Set<String> aliases = getAliases(nd, owi);
       // we pass null for aliases here because mergeWithChildrenPred filters
       // aliases in the children node context and we need to filter them in
       // the current JoinOperator's context
-      boolean hasUnpushedPredicates =
-          mergeWithChildrenPred(nd, owi, null, null, false);
+      mergeWithChildrenPred(nd, owi, null, null);
       ExprWalkerInfo prunePreds =
           owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd);
       if (prunePreds != null) {
@@ -255,31 +288,42 @@ public final class OpProcFactory {
           }
           prunePreds.getFinalCandidates().remove(alias);
         }
-        if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
-            HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) {
-          applyFilterTransitivity((JoinOperator) nd, owi);
-        }
-        if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
-            HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
-          // Here, we add all the "non-final candidiates", ie. the predicates
-          // rejected from pushdown through this operator to unpushedPreds
-          // and pass it to createFilter
-          ExprWalkerInfo unpushedPreds = new ExprWalkerInfo();
-          for (Entry<String, List<ExprNodeDesc>> entry :
-            prunePreds.getNonFinalCandidates().entrySet()) {
-            for (ExprNodeDesc expr : entry.getValue()) {
-              assert prunePreds.getNewToOldExprMap().containsKey(expr);
-              ExprNodeDesc oldExpr = prunePreds.getNewToOldExprMap().get(expr);
-              unpushedPreds.addAlias(oldExpr, entry.getKey());
-              unpushedPreds.addFinalCandidate(oldExpr);
-            }
-          }
-          return createFilter((Operator)nd, unpushedPreds, owi);
-        }
+        return handlePredicates(nd, prunePreds, owi);
       }
       return null;
     }
 
+    protected Set<String> getAliases(Node nd, OpWalkerInfo owi) throws SemanticException {
+      return owi.getRowResolver(nd).getTableNames();
+    }
+
+    protected Object handlePredicates(Node nd, ExprWalkerInfo prunePreds, OpWalkerInfo owi)
+        throws SemanticException {
+      if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+          HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
+        return createFilter((Operator)nd, prunePreds.getResidualPredicates(true), owi);
+      }
+      return null;
+    }
+  }
+
+  public static class JoinPPD extends JoinerPPD {
+
+    @Override
+    protected Set<String> getAliases(Node nd, OpWalkerInfo owi) {
+      return getQualifiedAliases((JoinOperator) nd, owi.getRowResolver(nd));
+    }
+
+    @Override
+    protected Object handlePredicates(Node nd, ExprWalkerInfo prunePreds, OpWalkerInfo owi)
+        throws SemanticException {
+      if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+          HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) {
+        applyFilterTransitivity((JoinOperator) nd, owi);
+      }
+      return super.handlePredicates(nd, prunePreds, owi);
+    }
+
     /**
      * Adds additional pushdown predicates for a join operator by replicating
      * filters transitively over all the equijoin conditions.
@@ -495,66 +539,56 @@ public final class OpProcFactory {
   }
 
   /**
-   * Processor for ReduceSink operator.
-   *
+   * Default processor which just merges its children.
    */
-  public static class ReduceSinkPPD extends DefaultPPD implements NodeProcessor {
+  public static class DefaultPPD implements NodeProcessor {
+
     @Override
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
       LOG.info("Processing for " + nd.getName() + "("
           + ((Operator) nd).getIdentifier() + ")");
-      ReduceSinkOperator rs = (ReduceSinkOperator) nd;
       OpWalkerInfo owi = (OpWalkerInfo) procCtx;
 
-      Set<String> aliases;
-      boolean ignoreAliases = false;
-      if (rs.getInputAlias() != null) {
-        aliases = new HashSet<String>(Arrays.asList(rs.getInputAlias()));
-      } else {
-        aliases = owi.getRowResolver(nd).getTableNames();
-        if (aliases.size() == 1 && aliases.contains("")) {
-          // Reduce sink of group by operator
-          ignoreAliases = true;
-        }
-      }
-      boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, aliases, ignoreAliases);
-      if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+      Set<String> includes = getQualifiedAliases((Operator<?>) nd, owi);
+      boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, includes);
+      if (hasUnpushedPredicates && HiveConf.getBoolVar(owi.getParseContext().getConf(),
           HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
-        if (hasUnpushedPredicates) {
-          Operator<? extends OperatorDesc> op =
-            (Operator<? extends OperatorDesc>) nd;
-          Operator<? extends OperatorDesc> childOperator = op.getChildOperators().get(0);
-          if(childOperator.getParentOperators().size()==1) {
-            owi.getCandidateFilterOps().clear();
+        if (includes != null || nd instanceof ReduceSinkOperator) {
+          owi.getCandidateFilterOps().clear();
+        } else {
+          ExprWalkerInfo pruned = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd);
+          Map<String, List<ExprNodeDesc>> residual = pruned.getResidualPredicates(true);
+          if (residual != null && !residual.isEmpty()) {
+            createFilter((Operator) nd, residual, owi);
+            pruned.getNonFinalCandidates().clear();
           }
         }
       }
       return null;
     }
 
-  }
-
-  /**
-   * Default processor which just merges its children.
-   */
-  public static class DefaultPPD implements NodeProcessor {
-
-    @Override
-    public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
-        Object... nodeOutputs) throws SemanticException {
-      LOG.info("Processing for " + nd.getName() + "("
-          + ((Operator) nd).getIdentifier() + ")");
-      OpWalkerInfo owi = (OpWalkerInfo) procCtx;
-      boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, null, false);
-      if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
-          HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
-        if (hasUnpushedPredicates) {
-          ExprWalkerInfo unpushedPreds = mergeChildrenPred(nd, owi, null, false);
-          return createFilter((Operator)nd, unpushedPreds, owi);
-        }
+    // RS for join, SEL(*) for lateral view
+    // SEL for union does not count (should be copied to both sides)
+    private Set<String> getQualifiedAliases(Operator<?> operator, OpWalkerInfo owi) {
+      if (operator.getNumChild() != 1) {
+        return null;
       }
-      return null;
+      Operator<?> child = operator.getChildOperators().get(0);
+      if (!(child instanceof JoinOperator || child instanceof LateralViewJoinOperator)) {
+        return null;
+      }
+      if (operator instanceof ReduceSinkOperator &&
+          ((ReduceSinkOperator)operator).getInputAliases() != null) {
+        String[] aliases = ((ReduceSinkOperator)operator).getInputAliases();
+        return new HashSet<String>(Arrays.asList(aliases));
+      }
+      Set<String> includes = owi.getRowResolver(operator).getTableNames();
+      if (includes.size() == 1 && includes.contains("")) {
+        // Reduce sink of group by operator
+        return null;
+      }
+      return includes;
     }
 
     /**
@@ -585,39 +619,22 @@ public final class OpProcFactory {
      * @param aliases
      *          aliases that this operator can pushdown. null means that all
      *          aliases can be pushed down
-     * @param ignoreAliases
      * @throws SemanticException
      */
     protected boolean mergeWithChildrenPred(Node nd, OpWalkerInfo owi,
-        ExprWalkerInfo ewi, Set<String> aliases, boolean ignoreAliases)
-        throws SemanticException {
-      boolean hasUnpushedPredicates = false;
-      Operator<?> current = (Operator<?>) nd;
-      List<Operator<?>> children = current.getChildOperators();
-      if (children == null || children.isEmpty()) {
-        return hasUnpushedPredicates;
-      }
-      if (children.size() > 1) {
-        // ppd for multi-insert query is not yet implemented
-        // no-op for leafs
-        for (Operator<?> child : children) {
-          removeCandidates(child, owi); // remove candidated filters on this branch
-        }
-        return hasUnpushedPredicates;
-      }
-      Operator<? extends OperatorDesc> op =
-        (Operator<? extends OperatorDesc>) nd;
-      ExprWalkerInfo childPreds = owi.getPrunedPreds(children.get(0));
+        ExprWalkerInfo ewi, Set<String> aliases) throws SemanticException {
+      Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
+      ExprWalkerInfo childPreds = getChildWalkerInfo(op, owi);
       if (childPreds == null) {
-        return hasUnpushedPredicates;
+        return false;
       }
       if (ewi == null) {
         ewi = new ExprWalkerInfo();
       }
+      boolean hasUnpushedPredicates = false;
       for (Entry<String, List<ExprNodeDesc>> e : childPreds
           .getFinalCandidates().entrySet()) {
-        if (ignoreAliases || aliases == null || aliases.contains(e.getKey())
-            || e.getKey() == null) {
+        if (aliases == null || e.getKey() == null || aliases.contains(e.getKey())) {
           // e.getKey() (alias) can be null in case of constant expressions. see
           // input8.q
           ExprWalkerInfo extractPushdownPreds = ExprWalkerProcFactory
@@ -627,25 +644,12 @@ public final class OpProcFactory {
           }
           ewi.merge(extractPushdownPreds);
           logExpr(nd, extractPushdownPreds);
-        } else {
-          hasUnpushedPredicates = true;
         }
       }
       owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, ewi);
       return hasUnpushedPredicates;
     }
 
-    private void removeCandidates(Operator<?> operator, OpWalkerInfo owi) {
-      if (operator instanceof FilterOperator) {
-        owi.getCandidateFilterOps().remove(operator);
-      }
-      if (operator.getChildOperators() != null) {
-        for (Operator<?> child : operator.getChildOperators()) {
-          removeCandidates(child, owi);
-        }
-      }
-    }
-
     protected ExprWalkerInfo mergeChildrenPred(Node nd, OpWalkerInfo owi,
         Set<String> excludedAliases, boolean ignoreAliases)
         throws SemanticException {
@@ -678,13 +682,16 @@ public final class OpProcFactory {
         || pushDownPreds.getFinalCandidates().size() == 0) {
       return null;
     }
+    return createFilter(op, pushDownPreds.getFinalCandidates(), owi);
+  }
 
+  protected static Object createFilter(Operator op,
+      Map<String, List<ExprNodeDesc>> predicates, OpWalkerInfo owi) {
     RowResolver inputRR = owi.getRowResolver(op);
 
     // combine all predicates into a single expression
     List<ExprNodeDesc> preds = new ArrayList<ExprNodeDesc>();
-    Iterator<List<ExprNodeDesc>> iterator = pushDownPreds.getFinalCandidates()
-        .values().iterator();
+    Iterator<List<ExprNodeDesc>> iterator = predicates.values().iterator();
     while (iterator.hasNext()) {
       for (ExprNodeDesc pred : iterator.next()) {
         preds = ExprNodeDescUtils.split(pred, preds);
@@ -831,8 +838,8 @@ public final class OpProcFactory {
       }
       if (decomposed.residualPredicate != null) {
         LOG.debug(
-          "Residual predicate:  "
-          + decomposed.residualPredicate.getExprString());
+            "Residual predicate:  "
+                + decomposed.residualPredicate.getExprString());
       }
     }
     tableScanDesc.setFilterExpr(decomposed.pushedPredicate);
@@ -847,10 +854,6 @@ public final class OpProcFactory {
     return new JoinPPD();
   }
 
-  public static NodeProcessor getRSProc() {
-    return new ReduceSinkPPD();
-  }
-
   public static NodeProcessor getTSProc() {
     return new TableScanPPD();
   }
@@ -871,12 +874,16 @@ public final class OpProcFactory {
     return new ScriptPPD();
   }
 
+  public static NodeProcessor getLVFProc() {
+    return new LateralViewForwardPPD();
+  }
+
   public static NodeProcessor getUDTFProc() {
     return new UDTFPPD();
   }
 
-  public static NodeProcessor getLVFProc() {
-    return new LateralViewForwardPPD();
+  public static NodeProcessor getLVJProc() {
+    return new JoinerPPD();
   }
 
   private OpProcFactory() {

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java Sat Mar 22 05:03:41 2014
@@ -24,9 +24,9 @@ import java.util.Map;
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
+import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
 import org.apache.hadoop.hive.ql.exec.LimitOperator;
 import org.apache.hadoop.hive.ql.exec.PTFOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
 import org.apache.hadoop.hive.ql.exec.ScriptOperator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.UDTFOperator;
@@ -91,18 +91,15 @@ public class PredicatePushDown implement
       FilterOperator.getOperatorName() + "%"),
       OpProcFactory.getFilterProc());
     opRules.put(new RuleRegExp("R2",
-        PTFOperator.getOperatorName() + "%"),
-        OpProcFactory.getPTFProc());
+      PTFOperator.getOperatorName() + "%"),
+      OpProcFactory.getPTFProc());
     opRules.put(new RuleRegExp("R3",
       CommonJoinOperator.getOperatorName() + "%"),
       OpProcFactory.getJoinProc());
     opRules.put(new RuleRegExp("R4",
-      ReduceSinkOperator.getOperatorName() + "%"),
-      OpProcFactory.getRSProc());
-    opRules.put(new RuleRegExp("R5",
       TableScanOperator.getOperatorName() + "%"),
       OpProcFactory.getTSProc());
-    opRules.put(new RuleRegExp("R6",
+    opRules.put(new RuleRegExp("R5",
       ScriptOperator.getOperatorName() + "%"),
       OpProcFactory.getSCRProc());
     opRules.put(new RuleRegExp("R6",
@@ -114,6 +111,9 @@ public class PredicatePushDown implement
     opRules.put(new RuleRegExp("R8",
       LateralViewForwardOperator.getOperatorName() + "%"),
       OpProcFactory.getLVFProc());
+    opRules.put(new RuleRegExp("R9",
+      LateralViewJoinOperator.getOperatorName() + "%"),
+      OpProcFactory.getLVJProc());
 
     // The dispatcher fires the processor corresponding to the closest matching
     // rule and passes the context along

Modified: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/lateral_view_ppd.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/lateral_view_ppd.q?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/lateral_view_ppd.q (original)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/lateral_view_ppd.q Sat Mar 22 05:03:41 2014
@@ -11,3 +11,7 @@ SELECT value, myCol FROM (SELECT * FROM 
 
 EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE key='0';
 SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE key='0';
+
+-- HIVE-4293 Predicates following UDTF operator are removed by PPD
+EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0';
+SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0';
\ No newline at end of file

Added: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_join4.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_join4.q?rev=1580169&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_join4.q (added)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_join4.q Sat Mar 22 05:03:41 2014
@@ -0,0 +1,22 @@
+create table dual(a string);
+
+set hive.optimize.ppd=true;
+drop table if exists test_tbl ;
+
+create table test_tbl (id string,name string);
+
+insert into table test_tbl
+select 'a','b' from dual;
+
+explain
+select t2.* 
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a';
+
+select t2.* 
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a';
\ No newline at end of file

Modified: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_transform.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_transform.q?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_transform.q (original)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_transform.q Sat Mar 22 05:03:41 2014
@@ -36,3 +36,12 @@ FROM (
   CLUSTER BY tkey 
 ) tmap
 SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100;
+
+-- test described in HIVE-4598
+
+EXPLAIN
+FROM (
+    FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y
+) reduced
+insert overwrite local directory '/tmp/a' select * where x='a' or x='b'
+insert overwrite local directory '/tmp/b' select * where x='c' or x='d';

Added: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_udtf.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_udtf.q?rev=1580169&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_udtf.q (added)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/ppd_udtf.q Sat Mar 22 05:03:41 2014
@@ -0,0 +1,12 @@
+explain
+SELECT value from (
+  select explode(array(key, value)) as (value) from (
+    select * FROM src WHERE key > 400
+  ) A
+) B WHERE value < 450;
+
+SELECT value from (
+  select explode(array(key, value)) as (value) from (
+    select * FROM src WHERE key > 400
+  ) A
+) B WHERE value < 450;

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/cluster.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/cluster.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/cluster.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/cluster.q.out Sat Mar 22 05:03:41 2014
@@ -393,33 +393,33 @@ STAGE PLANS:
           TableScan
             alias: x
             Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: key (type: string), value (type: string)
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-              Reduce Output Operator
-                key expressions: _col0 (type: string)
-                sort order: +
-                Map-reduce partition columns: _col0 (type: string)
-                Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-                value expressions: _col0 (type: string), _col1 (type: string)
+            Filter Operator
+              predicate: (key = 20) (type: boolean)
+              Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                expressions: key (type: string), value (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: string)
+                  sort order: +
+                  Map-reduce partition columns: _col0 (type: string)
+                  Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+                  value expressions: _col0 (type: string), _col1 (type: string)
       Reduce Operator Tree:
         Extract
-          Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
-          Filter Operator
-            predicate: (_col0 = 20) (type: boolean)
+          Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+          Select Operator
+            expressions: _col0 (type: string), _col1 (type: string)
+            outputColumnNames: _col0, _col1
             Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              expressions: _col0 (type: string), _col1 (type: string)
-              outputColumnNames: _col0, _col1
+            File Output Operator
+              compressed: false
               Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
-              File Output Operator
-                compressed: false
-                Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
-                table:
-                    input format: org.apache.hadoop.mapred.TextInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ctas_colname.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ctas_colname.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ctas_colname.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ctas_colname.q.out Sat Mar 22 05:03:41 2014
@@ -1135,15 +1135,15 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1
           Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((_col0 < 9) and _col1 is not null) (type: boolean)
-            Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+            predicate: _col1 is not null (type: boolean)
+            Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               expressions: _col1 (type: string), _col0 (type: string)
               outputColumnNames: _col0, _col1
-              Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
               File Output Operator
                 compressed: false
-                Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
                 table:
                     input format: org.apache.hadoop.mapred.TextInputFormat
                     output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/lateral_view_ppd.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/lateral_view_ppd.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/lateral_view_ppd.q.out Sat Mar 22 05:03:41 2014
@@ -100,26 +100,23 @@ STAGE PLANS:
               Lateral View Forward
                 Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
                 Select Operator
-                  expressions: key (type: string), value (type: string)
-                  outputColumnNames: key, value
+                  expressions: value (type: string)
+                  outputColumnNames: value
                   Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
                   Lateral View Join Operator
-                    outputColumnNames: _col0, _col1, _col4
-                    Statistics: Num rows: 28 Data size: 5610 Basic stats: COMPLETE Column stats: NONE
-                    Filter Operator
-                      predicate: ((_col0 = '0') and (_col4 = 1)) (type: boolean)
-                      Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
-                      Select Operator
-                        expressions: _col1 (type: string), _col4 (type: int)
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
-                        File Output Operator
-                          compressed: false
-                          Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    outputColumnNames: _col1, _col4
+                    Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: _col1 (type: string), _col4 (type: int)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
+                      File Output Operator
+                        compressed: false
+                        Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
+                        table:
+                            input format: org.apache.hadoop.mapred.TextInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 Select Operator
                   expressions: array(1,2,3) (type: array<int>)
                   outputColumnNames: _col0
@@ -127,19 +124,19 @@ STAGE PLANS:
                   UDTF Operator
                     Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
                     function name: explode
-                    Lateral View Join Operator
-                      outputColumnNames: _col0, _col1, _col4
-                      Statistics: Num rows: 28 Data size: 5610 Basic stats: COMPLETE Column stats: NONE
-                      Filter Operator
-                        predicate: ((_col0 = '0') and (_col4 = 1)) (type: boolean)
-                        Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+                    Filter Operator
+                      predicate: (col = 1) (type: boolean)
+                      Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+                      Lateral View Join Operator
+                        outputColumnNames: _col1, _col4
+                        Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
                         Select Operator
                           expressions: _col1 (type: string), _col4 (type: int)
                           outputColumnNames: _col0, _col1
-                          Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
                           File Output Operator
                             compressed: false
-                            Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+                            Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
                             table:
                                 input format: org.apache.hadoop.mapred.TextInputFormat
                                 output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -408,3 +405,86 @@ val_0	2
 val_0	3
 val_0	3
 val_0	3
+PREHOOK: query: -- HIVE-4293 Predicates following UDTF operator are removed by PPD
+EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- HIVE-4293 Predicates following UDTF operator are removed by PPD
+EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src
+            Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (key = '0') (type: boolean)
+              Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+              Lateral View Forward
+                Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+                Select Operator
+                  expressions: value (type: string)
+                  outputColumnNames: value
+                  Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+                  Lateral View Join Operator
+                    outputColumnNames: _col1, _col4
+                    Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: _col1 (type: string), _col4 (type: int)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+                      File Output Operator
+                        compressed: false
+                        Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+                        table:
+                            input format: org.apache.hadoop.mapred.TextInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                Select Operator
+                  expressions: array(1,2,3) (type: array<int>)
+                  outputColumnNames: _col0
+                  Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+                  UDTF Operator
+                    Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+                    function name: explode
+                    Filter Operator
+                      predicate: (col > 1) (type: boolean)
+                      Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
+                      Lateral View Join Operator
+                        outputColumnNames: _col1, _col4
+                        Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+                        Select Operator
+                          expressions: _col1 (type: string), _col4 (type: int)
+                          outputColumnNames: _col0, _col1
+                          Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+                          File Output Operator
+                            compressed: false
+                            Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+                            table:
+                                input format: org.apache.hadoop.mapred.TextInputFormat
+                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+PREHOOK: query: SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+val_0	2
+val_0	3
+val_0	2
+val_0	3
+val_0	2
+val_0	3

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd2.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd2.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd2.q.out Sat Mar 22 05:03:41 2014
@@ -60,24 +60,20 @@ STAGE PLANS:
           mode: mergepartial
           outputColumnNames: _col0, _col1
           Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
-          Select Operator
-            expressions: _col0 (type: string), _col1 (type: bigint)
-            outputColumnNames: _col0, _col1
-            Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
-            Filter Operator
-              predicate: (_col1 > 1) (type: boolean)
+          Filter Operator
+            predicate: (_col1 > 1) (type: boolean)
+            Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: _col0 (type: string), _col1 (type: bigint)
+              outputColumnNames: _col0, _col1
               Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
-              Select Operator
-                expressions: _col0 (type: string), _col1 (type: bigint)
-                outputColumnNames: _col0, _col1
+              File Output Operator
+                compressed: false
                 Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
-                File Output Operator
-                  compressed: false
-                  Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
-                  table:
-                      input format: org.apache.hadoop.mapred.TextInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby.q.out Sat Mar 22 05:03:41 2014
@@ -263,7 +263,7 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1
           Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((_col0 > 'val_200') and ((_col1 > 30) or (_col0 < 'val_400'))) (type: boolean)
+            predicate: ((_col1 > 30) or (_col0 < 'val_400')) (type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Select Operator
               expressions: _col0 (type: string)

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby2.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby2.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_gby2.q.out Sat Mar 22 05:03:41 2014
@@ -181,7 +181,7 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1
           Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: ((_col0 > 'val_200') and ((_col1 > 30) or (_col0 < 'val_400'))) (type: boolean)
+            predicate: ((_col1 > 30) or (_col0 < 'val_400')) (type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Select Operator
               expressions: _col0 (type: string), _col1 (type: bigint)

Added: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_join4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_join4.q.out?rev=1580169&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_join4.q.out (added)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_join4.q.out Sat Mar 22 05:03:41 2014
@@ -0,0 +1,144 @@
+PREHOOK: query: create table dual(a string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table dual(a string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dual
+PREHOOK: query: drop table if exists test_tbl
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists test_tbl
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table test_tbl (id string,name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table test_tbl (id string,name string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_tbl
+PREHOOK: query: insert into table test_tbl
+select 'a','b' from dual
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dual
+PREHOOK: Output: default@test_tbl
+POSTHOOK: query: insert into table test_tbl
+select 'a','b' from dual
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dual
+POSTHOOK: Output: default@test_tbl
+POSTHOOK: Lineage: test_tbl.id SIMPLE []
+POSTHOOK: Lineage: test_tbl.name SIMPLE []
+PREHOOK: query: explain
+select t2.* 
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select t2.* 
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: test_tbl.id SIMPLE []
+POSTHOOK: Lineage: test_tbl.name SIMPLE []
+STAGE DEPENDENCIES:
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-2
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: test_tbl
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            Filter Operator
+              predicate: ((name = 'c') and (id = 'a')) (type: boolean)
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              Select Operator
+                expressions: id (type: string), name (type: string)
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: string)
+                  sort order: +
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  value expressions: _col0 (type: string), _col1 (type: string)
+      Reduce Operator Tree:
+        Extract
+          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: t3
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            Filter Operator
+              predicate: (id = 'a') (type: boolean)
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              Reduce Output Operator
+                key expressions: id (type: string)
+                sort order: +
+                Map-reduce partition columns: id (type: string)
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+          TableScan
+            Reduce Output Operator
+              key expressions: _col0 (type: string)
+              sort order: +
+              Map-reduce partition columns: _col0 (type: string)
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              value expressions: _col0 (type: string), _col1 (type: string)
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 {VALUE._col0} {VALUE._col1}
+            1 
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+          Select Operator
+            expressions: _col0 (type: string), _col1 (type: string)
+            outputColumnNames: _col0, _col1
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+            File Output Operator
+              compressed: false
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+PREHOOK: query: select t2.* 
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_tbl
+#### A masked pattern was here ####
+POSTHOOK: query: select t2.* 
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_tbl
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test_tbl.id SIMPLE []
+POSTHOOK: Lineage: test_tbl.name SIMPLE []

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_transform.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_transform.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_transform.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_transform.q.out Sat Mar 22 05:03:41 2014
@@ -342,3 +342,83 @@ POSTHOOK: Input: default@src
 97	val_97
 98	val_98
 98	val_98
+PREHOOK: query: -- test described in HIVE-4598
+
+EXPLAIN
+FROM (
+    FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y
+) reduced
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+POSTHOOK: query: -- test described in HIVE-4598
+
+EXPLAIN
+FROM (
+    FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y
+) reduced
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-2 is a root stage
+  Stage-0 depends on stages: Stage-2
+  Stage-1 depends on stages: Stage-2
+
+STAGE PLANS:
+  Stage: Stage-2
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src
+            Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+            Select Operator
+              expressions: key (type: string), value (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+              Transform Operator
+                command: cat
+                output info:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: ((_col0 = 'a') or (_col0 = 'b')) (type: boolean)
+                  Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                Filter Operator
+                  predicate: ((_col0 = 'c') or (_col0 = 'd')) (type: boolean)
+                  Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Move Operator
+      files:
+          hdfs directory: false
+#### A masked pattern was here ####
+
+  Stage: Stage-1
+    Move Operator
+      files:
+          hdfs directory: false
+#### A masked pattern was here ####
+

Added: hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_udtf.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_udtf.q.out?rev=1580169&view=auto
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_udtf.q.out (added)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/ppd_udtf.q.out Sat Mar 22 05:03:41 2014
@@ -0,0 +1,125 @@
+PREHOOK: query: explain
+SELECT value from (
+  select explode(array(key, value)) as (value) from (
+    select * FROM src WHERE key > 400
+  ) A
+) B WHERE value < 450
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT value from (
+  select explode(array(key, value)) as (value) from (
+    select * FROM src WHERE key > 400
+  ) A
+) B WHERE value < 450
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src
+            Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+            Filter Operator
+              predicate: (key > 400) (type: boolean)
+              Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
+              Select Operator
+                expressions: array(key,value) (type: array<string>)
+                outputColumnNames: _col0
+                Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
+                UDTF Operator
+                  Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
+                  function name: explode
+                  Filter Operator
+                    predicate: (col < 450) (type: boolean)
+                    Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: col (type: string)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE
+                      File Output Operator
+                        compressed: false
+                        Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE
+                        table:
+                            input format: org.apache.hadoop.mapred.TextInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+
+PREHOOK: query: SELECT value from (
+  select explode(array(key, value)) as (value) from (
+    select * FROM src WHERE key > 400
+  ) A
+) B WHERE value < 450
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value from (
+  select explode(array(key, value)) as (value) from (
+    select * FROM src WHERE key > 400
+  ) A
+) B WHERE value < 450
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+409
+401
+406
+429
+403
+417
+430
+446
+413
+417
+439
+438
+427
+437
+430
+404
+411
+431
+418
+404
+436
+409
+413
+419
+435
+401
+402
+430
+449
+401
+438
+432
+403
+406
+409
+406
+401
+424
+431
+431
+424
+438
+414
+439
+417
+444
+429
+443
+406
+401
+421
+407
+448
+414
+403

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/subquery_notin_having.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/subquery_notin_having.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/subquery_notin_having.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/subquery_notin_having.q.out Sat Mar 22 05:03:41 2014
@@ -440,7 +440,7 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1, _col2, _col3
           Statistics: Num rows: 14 Data size: 1531 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: (((_col2 - _col1) > 600) and (_col0 is null or _col1 is null)) (type: boolean)
+            predicate: ((_col0 is null or _col1 is null) and ((_col2 - _col1) > 600)) (type: boolean)
             Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
             Select Operator
               Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
@@ -598,7 +598,7 @@ STAGE PLANS:
           TableScan
             Reduce Output Operator
               sort order: 
-              Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
@@ -689,10 +689,10 @@ STAGE PLANS:
           outputColumnNames: _col0, _col1, _col2
           Statistics: Num rows: 7 Data size: 765 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
-            predicate: (((_col1 - _col2) > 600) and _col0 is null) (type: boolean)
-            Statistics: Num rows: 1 Data size: 109 Basic stats: COMPLETE Column stats: NONE
+            predicate: ((_col1 - _col2) > 600) (type: boolean)
+            Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
             Select Operator
-              Statistics: Num rows: 1 Data size: 109 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
               Group By Operator
                 aggregations: count()
                 mode: hash
@@ -721,16 +721,16 @@ STAGE PLANS:
           Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
           Filter Operator
             predicate: (_col0 = 0) (type: boolean)
-            Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Select Operator
               expressions: _col0 (type: bigint)
               outputColumnNames: _col0
-              Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Group By Operator
                 keys: _col0 (type: bigint)
                 mode: hash
                 outputColumnNames: _col0
-                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
                 File Output Operator
                   compressed: false
                   table:

Modified: hive/branches/branch-0.13/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/results/clientpositive/udtf_json_tuple.q.out?rev=1580169&r1=1580168&r2=1580169&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/results/clientpositive/udtf_json_tuple.q.out (original)
+++ hive/branches/branch-0.13/ql/src/test/results/clientpositive/udtf_json_tuple.q.out Sat Mar 22 05:03:41 2014
@@ -303,26 +303,23 @@ STAGE PLANS:
                 Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
                 Lateral View Join Operator
                   outputColumnNames: _col4, _col5, _col6, _col7, _col8
-                  Statistics: Num rows: 12 Data size: 472 Basic stats: COMPLETE Column stats: NONE
-                  Filter Operator
-                    predicate: _col4 is not null (type: boolean)
-                    Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
-                    Select Operator
-                      expressions: _col5 (type: string)
-                      outputColumnNames: _col5
-                      Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
-                      Group By Operator
-                        aggregations: count()
-                        keys: _col5 (type: string)
-                        mode: hash
-                        outputColumnNames: _col0, _col1
-                        Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
-                        Reduce Output Operator
-                          key expressions: _col0 (type: string)
-                          sort order: +
-                          Map-reduce partition columns: _col0 (type: string)
-                          Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
-                          value expressions: _col1 (type: bigint)
+                  Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col5 (type: string)
+                    outputColumnNames: _col5
+                    Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+                    Group By Operator
+                      aggregations: count()
+                      keys: _col5 (type: string)
+                      mode: hash
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: bigint)
               Select Operator
                 expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string)
                 outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -330,27 +327,27 @@ STAGE PLANS:
                 UDTF Operator
                   Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
                   function name: json_tuple
-                  Lateral View Join Operator
-                    outputColumnNames: _col4, _col5, _col6, _col7, _col8
-                    Statistics: Num rows: 12 Data size: 472 Basic stats: COMPLETE Column stats: NONE
-                    Filter Operator
-                      predicate: _col4 is not null (type: boolean)
-                      Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: c0 is not null (type: boolean)
+                    Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+                    Lateral View Join Operator
+                      outputColumnNames: _col4, _col5, _col6, _col7, _col8
+                      Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col5 (type: string)
                         outputColumnNames: _col5
-                        Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+                        Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
                         Group By Operator
                           aggregations: count()
                           keys: _col5 (type: string)
                           mode: hash
                           outputColumnNames: _col0, _col1
-                          Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+                          Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
                           Reduce Output Operator
                             key expressions: _col0 (type: string)
                             sort order: +
                             Map-reduce partition columns: _col0 (type: string)
-                            Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+                            Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
                             value expressions: _col1 (type: bigint)
       Reduce Operator Tree:
         Group By Operator
@@ -358,11 +355,11 @@ STAGE PLANS:
           keys: KEY._col0 (type: string)
           mode: mergepartial
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
           Select Operator
             expressions: _col0 (type: string), _col1 (type: bigint)
             outputColumnNames: _col0, _col1
-            Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
             File Output Operator
               compressed: false
               table:
@@ -377,14 +374,14 @@ STAGE PLANS:
             Reduce Output Operator
               key expressions: _col0 (type: string)
               sort order: +
-              Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+              Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
               value expressions: _col0 (type: string), _col1 (type: bigint)
       Reduce Operator Tree:
         Extract
-          Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
-            Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+            Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
             table:
                 input format: org.apache.hadoop.mapred.TextInputFormat
                 output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat



Mime
View raw message