hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1619294 - in /hive/branches/cbo/ql/src: java/org/apache/hadoop/hive/ql/optimizer/optiq/ java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/ java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ java/org/apache/hadoop/hive/ql/parse/ ...
Date Thu, 21 Aug 2014 05:40:49 GMT
Author: hashutosh
Date: Thu Aug 21 05:40:48 2014
New Revision: 1619294

URL: http://svn.apache.org/r1619294
Log:
HIVE-7804 : [CBO] Support semi-joins (Harish Butani)

Modified:
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
    hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java?rev=1619294&r1=1619293&r2=1619294&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/HiveOptiqUtil.java Thu Aug 21 05:40:48 2014
@@ -4,8 +4,16 @@ import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
 
+import org.eigenbase.rel.RelFactories.ProjectFactory;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeField;
+import org.eigenbase.rex.RexBuilder;
 import org.eigenbase.rex.RexInputRef;
 import org.eigenbase.rex.RexNode;
+import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.sql.validate.SqlValidatorUtil;
+import org.eigenbase.util.Pair;
 
 /**
  * Generic utility functions needed for Optiq based Hive CBO.
@@ -48,4 +56,132 @@ public class HiveOptiqUtil {
   @Deprecated
   public static void todo(String s) {
   }
+
+  /**
+   * Push any equi join conditions that are not column references as Projections
+   * on top of the children.
+   * 
+   * @param factory
+   *          Project factory to use.
+   * @param inputRels
+   *          inputs to a join
+   * @param leftJoinKeys
+   *          expressions for LHS of join key
+   * @param rightJoinKeys
+   *          expressions for RHS of join key
+   * @param systemColCount
+   *          number of system columns, usually zero. These columns are
+   *          projected at the leading edge of the output row.
+   * @param leftKeys       on return this contains the join key positions from
+   *                       the new project rel on the LHS.
+   * @param rightKeys      on return this contains the join key positions from
+   *                       the new project rel on the RHS.
+   * @return the join condition after the equi expressions pushed down.
+   */
+  public static RexNode projectNonColumnEquiConditions(ProjectFactory factory,
+      RelNode[] inputRels, List<RexNode> leftJoinKeys,
+      List<RexNode> rightJoinKeys, int systemColCount, List<Integer> leftKeys,
+      List<Integer> rightKeys) {
+    RelNode leftRel = inputRels[0];
+    RelNode rightRel = inputRels[1];
+    RexBuilder rexBuilder = leftRel.getCluster().getRexBuilder();
+    RexNode outJoinCond = null;
+
+    int origLeftInputSize = leftRel.getRowType().getFieldCount();
+    int origRightInputSize = rightRel.getRowType().getFieldCount();
+
+    List<RexNode> newLeftFields = new ArrayList<RexNode>();
+    List<String> newLeftFieldNames = new ArrayList<String>();
+
+    List<RexNode> newRightFields = new ArrayList<RexNode>();
+    List<String> newRightFieldNames = new ArrayList<String>();
+    int leftKeyCount = leftJoinKeys.size();
+    int i;
+
+    for (i = 0; i < origLeftInputSize; i++) {
+      final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i);
+      newLeftFields.add(rexBuilder.makeInputRef(field.getType(), i));
+      newLeftFieldNames.add(field.getName());
+    }
+
+    for (i = 0; i < origRightInputSize; i++) {
+      final RelDataTypeField field = rightRel.getRowType().getFieldList()
+          .get(i);
+      newRightFields.add(rexBuilder.makeInputRef(field.getType(), i));
+      newRightFieldNames.add(field.getName());
+    }
+
+    int newKeyCount = 0;
+    List<Pair<Integer, Integer>> origColEqConds = new ArrayList<Pair<Integer, Integer>>();
+    for (i = 0; i < leftKeyCount; i++) {
+      RexNode leftKey = leftJoinKeys.get(i);
+      RexNode rightKey = rightJoinKeys.get(i);
+
+      if (leftKey instanceof RexInputRef && rightKey instanceof RexInputRef) {
+        origColEqConds.add(Pair.of(((RexInputRef) leftKey).getIndex(),
+            ((RexInputRef) rightKey).getIndex()));
+      } else {
+        newLeftFields.add(leftKey);
+        newLeftFieldNames.add(null);
+        newRightFields.add(rightKey);
+        newRightFieldNames.add(null);
+        newKeyCount++;
+      }
+    }
+
+    for (i = 0; i < origColEqConds.size(); i++) {
+      Pair<Integer, Integer> p = origColEqConds.get(i);
+      RexNode leftKey = leftJoinKeys.get(p.left);
+      RexNode rightKey = rightJoinKeys.get(p.right);
+      leftKeys.add(p.left);
+      rightKeys.add(p.right);
+      RexNode cond = rexBuilder.makeCall(
+          SqlStdOperatorTable.EQUALS,
+          rexBuilder.makeInputRef(leftKey.getType(), systemColCount + p.left),
+          rexBuilder.makeInputRef(rightKey.getType(), systemColCount
+              + origLeftInputSize + newKeyCount + p.right));
+      if (outJoinCond == null) {
+        outJoinCond = cond;
+      } else {
+        outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond,
+            cond);
+      }
+    }
+
+    if (newKeyCount == 0) {
+      return outJoinCond;
+    }
+
+    int newLeftOffset = systemColCount + origLeftInputSize;
+    int newRightOffset = systemColCount + origLeftInputSize
+        + origRightInputSize + newKeyCount;
+    for (i = 0; i < newKeyCount; i++) {
+      leftKeys.add(origLeftInputSize + i);
+      rightKeys.add(origRightInputSize + i);
+      RexNode cond = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, rexBuilder
+          .makeInputRef(newLeftFields.get(i).getType(), newLeftOffset + i),
+          rexBuilder.makeInputRef(newLeftFields.get(i).getType(),
+              newRightOffset + i));
+      if (outJoinCond == null) {
+        outJoinCond = cond;
+      } else {
+        outJoinCond = rexBuilder.makeCall(SqlStdOperatorTable.AND, outJoinCond,
+            cond);
+      }
+    }
+
+    // added project if need to produce new keys than the original input
+    // fields
+    if (newKeyCount > 0) {
+      leftRel = factory.createProject(leftRel, newLeftFields,
+          SqlValidatorUtil.uniquify(newLeftFieldNames));
+      rightRel = factory.createProject(rightRel, newRightFields,
+          SqlValidatorUtil.uniquify(newRightFieldNames));
+    }
+
+    inputRels[0] = leftRel;
+    inputRels[1] = rightRel;
+
+    return outJoinCond;
+  }
 }

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java?rev=1619294&r1=1619293&r2=1619294&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/rules/HiveRelFieldTrimmer.java Thu Aug 21 05:40:48 2014
@@ -24,6 +24,7 @@ import org.eigenbase.rel.AggregateRel;
 import org.eigenbase.rel.CalcRel;
 import org.eigenbase.rel.InvalidRelException;
 import org.eigenbase.rel.JoinRel;
+import org.eigenbase.rel.JoinRelBase;
 import org.eigenbase.rel.RelCollation;
 import org.eigenbase.rel.RelFieldCollation;
 import org.eigenbase.rel.RelNode;
@@ -33,6 +34,7 @@ import org.eigenbase.rel.TableFunctionRe
 import org.eigenbase.rel.TableModificationRel;
 import org.eigenbase.rel.ValuesRel;
 import org.eigenbase.rel.rules.RemoveTrivialProjectRule;
+import org.eigenbase.rel.rules.SemiJoinRel;
 import org.eigenbase.relopt.RelOptUtil;
 import org.eigenbase.reltype.RelDataType;
 import org.eigenbase.reltype.RelDataTypeField;
@@ -436,23 +438,20 @@ public class HiveRelFieldTrimmer impleme
 
   /**
    * Variant of {@link #trimFields(RelNode, BitSet, Set)} for {@link JoinRel}.
+   * 
+   * Have to do this because of the way ReflectUtil works. - if there is an
+   * exact match, things are fine. - otherwise it doesn't allow any ambiguity(in
+   * this case between a superClass(JoinRelBase) and an interface(HiveRel).
    */
-  public TrimResult trimFields(HiveJoinRel join, BitSet fieldsUsed,
+  private TrimResult _trimFields(JoinRelBase join, BitSet fieldsUsed,
       Set<RelDataTypeField> extraFields) {
     final RelDataType rowType = join.getRowType();
-    final int fieldCount = rowType.getFieldCount();
+    final int fieldCount = join.getSystemFieldList().size() +
+        join.getLeft().getRowType().getFieldCount() +
+        join.getRight().getRowType().getFieldCount();
     final RexNode conditionExpr = join.getCondition();
     final int systemFieldCount = join.getSystemFieldList().size();
 
-    /*
-     * todo: hb 6/26/14 for left SemiJoin we cannot trim yet.
-     * HiveJoinRelNode.deriveRowType return only left columns. Default field
-     * trimmer needs to be enhanced to handle this.
-     */
-    if (join.isLeftSemiJoin()) {
-      return new TrimResult(join, Mappings.createIdentity(fieldCount));
-    }
-
     // Add in fields used in the condition.
     BitSet fieldsUsedPlus = (BitSet) fieldsUsed.clone();
     final Set<RelDataTypeField> combinedInputExtraFields = new LinkedHashSet<RelDataTypeField>(
@@ -554,12 +553,39 @@ public class HiveRelFieldTrimmer impleme
         newInputs.get(0), newInputs.get(1));
     RexNode newConditionExpr = conditionExpr.accept(shuttle);
 
-    final HiveJoinRel newJoin = join.copy(join.getTraitSet(), newConditionExpr,
+    final JoinRelBase newJoin = join.copy(join.getTraitSet(), newConditionExpr,
         newInputs.get(0), newInputs.get(1), join.getJoinType(), false);
 
+    /*
+     * For SemiJoins only map fields from the left-side
+     */
+    if ( join instanceof SemiJoinRel ) {
+      Mapping inputMapping = inputMappings.get(0);
+      mapping = Mappings.create(MappingType.INVERSE_SURJECTION,
+          join.getRowType().getFieldCount(), newSystemFieldCount + inputMapping.getTargetCount());
+      for (int i = 0; i < newSystemFieldCount; ++i) {
+        mapping.set(i, i);
+      }
+      offset = systemFieldCount;
+      newOffset = newSystemFieldCount;
+      for (IntPair pair : inputMapping) {
+        mapping.set(pair.source + offset, pair.target + newOffset);
+      }
+    }
+
     return new TrimResult(newJoin, mapping);
   }
 
+  public TrimResult trimFields(HiveJoinRel join, BitSet fieldsUsed,
+      Set<RelDataTypeField> extraFields) {
+    return _trimFields((JoinRelBase) join, fieldsUsed, extraFields);
+  }
+  
+  public TrimResult trimFields(SemiJoinRel join, BitSet fieldsUsed,
+      Set<RelDataTypeField> extraFields) {
+    return _trimFields((JoinRelBase) join, fieldsUsed, extraFields);
+  }
+  
   /**
    * Variant of {@link #trimFields(RelNode, BitSet, Set)} for {@link SetOpRel}
    * (including UNION and UNION ALL).

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java?rev=1619294&r1=1619293&r2=1619294&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/translator/ASTConverter.java Thu Aug 21 05:40:48 2014
@@ -25,6 +25,7 @@ import org.eigenbase.rel.RelVisitor;
 import org.eigenbase.rel.SortRel;
 import org.eigenbase.rel.TableAccessRelBase;
 import org.eigenbase.rel.UnionRelBase;
+import org.eigenbase.rel.rules.SemiJoinRel;
 import org.eigenbase.reltype.RelDataTypeField;
 import org.eigenbase.rex.RexCall;
 import org.eigenbase.rex.RexFieldCollation;
@@ -187,8 +188,7 @@ public class ASTConverter {
       QueryBlockInfo right = convertSource(join.getRight());
       s = new Schema(left.schema, right.schema);
       ASTNode cond = join.getCondition().accept(new RexVisitor(s));
-      boolean semiJoin = ((join instanceof HiveJoinRel) && ((HiveJoinRel) join).isLeftSemiJoin()) ? true
-          : false;
+      boolean semiJoin = join instanceof SemiJoinRel;
       ast = ASTBuilder.join(left.ast, right.ast, join.getJoinType(), cond, semiJoin);
       if (semiJoin)
         s = left.schema;

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1619294&r1=1619293&r2=1619294&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Thu Aug 21 05:40:48 2014
@@ -110,6 +110,7 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.optimizer.Optimizer;
 import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
 import org.apache.hadoop.hive.ql.optimizer.optiq.HiveDefaultRelMetadataProvider;
+import org.apache.hadoop.hive.ql.optimizer.optiq.HiveOptiqUtil;
 import org.apache.hadoop.hive.ql.optimizer.optiq.Pair;
 import org.apache.hadoop.hive.ql.optimizer.optiq.RelOptHiveTable;
 import org.apache.hadoop.hive.ql.optimizer.optiq.TraitsUtil;
@@ -223,6 +224,8 @@ import org.apache.hadoop.mapred.InputFor
 import org.eigenbase.rel.AggregateCall;
 import org.eigenbase.rel.Aggregation;
 import org.eigenbase.rel.InvalidRelException;
+import org.eigenbase.rel.JoinInfo;
+import org.eigenbase.rel.JoinRelBase;
 import org.eigenbase.rel.JoinRelType;
 import org.eigenbase.rel.RelCollation;
 import org.eigenbase.rel.RelCollationImpl;
@@ -233,6 +236,7 @@ import org.eigenbase.rel.metadata.Chaine
 import org.eigenbase.rel.metadata.RelMetadataProvider;
 import org.eigenbase.rel.rules.ConvertMultiJoinRule;
 import org.eigenbase.rel.rules.LoptOptimizeJoinRule;
+import org.eigenbase.rel.rules.SemiJoinRel;
 import org.eigenbase.relopt.RelOptCluster;
 import org.eigenbase.relopt.RelOptPlanner;
 import org.eigenbase.relopt.RelOptQuery;
@@ -262,6 +266,7 @@ import org.eigenbase.sql.SqlKind;
 import org.eigenbase.sql.SqlNode;
 import org.eigenbase.sql.SqlLiteral;
 import org.eigenbase.util.CompositeList;
+import org.eigenbase.util.ImmutableIntList;
 
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
@@ -9579,7 +9584,10 @@ public class SemanticAnalyzer extends Ba
           throw new RuntimeException(
               "Couldn't do phase1 on CBO optimized query plan");
         }
-        prunedPartitions = ImmutableMap.copyOf(prunedPartitions);
+        // unfortunately making prunedPartitions immutable is not possible here
+        // with SemiJoins not all tables are costed in CBO,
+        // so their PartitionList is not evaluated until the run phase.
+        //prunedPartitions = ImmutableMap.copyOf(prunedPartitions);
         getMetaData(qb);
 
         disableJoinMerge = true;
@@ -12115,9 +12123,37 @@ public class SemanticAnalyzer extends Ba
         optiqJoinType = JoinRelType.INNER;
         break;
       }
-      joinRel = HiveJoinRel.getJoin(m_cluster, leftRel, rightRel,
-          optiqJoinCond, optiqJoinType, leftSemiJoin);
+      
+      if (leftSemiJoin) {
+        List<RelDataTypeField> sysFieldList = new ArrayList<RelDataTypeField>();
+        List<RexNode> leftJoinKeys = new ArrayList<RexNode>();
+        List<RexNode> rightJoinKeys = new ArrayList<RexNode>();
+
+        RexNode nonEquiConds = RelOptUtil.splitJoinCondition(sysFieldList,
+            leftRel, rightRel, optiqJoinCond, leftJoinKeys, rightJoinKeys,
+            null, null);
 
+        if (!nonEquiConds.isAlwaysTrue()) {
+          throw new SemanticException(
+              "Non equality condition not supported in Semi-Join"
+                  + nonEquiConds);
+        }
+
+        RelNode[] inputRels = new RelNode[] { leftRel, rightRel };
+        final List<Integer> leftKeys = new ArrayList<Integer>();
+        final List<Integer> rightKeys = new ArrayList<Integer>();
+        optiqJoinCond = HiveOptiqUtil.projectNonColumnEquiConditions(
+            HiveProjectRel.DEFAULT_PROJECT_FACTORY, inputRels, leftJoinKeys,
+            rightJoinKeys, 0, leftKeys, rightKeys);
+
+        joinRel = new SemiJoinRel(m_cluster,
+            m_cluster.traitSetOf(HiveRel.CONVENTION), inputRels[0],
+            inputRels[1], optiqJoinCond, ImmutableIntList.copyOf(leftKeys),
+            ImmutableIntList.copyOf(rightKeys));
+      } else {
+        joinRel = HiveJoinRel.getJoin(m_cluster, leftRel, rightRel,
+            optiqJoinCond, optiqJoinType, leftSemiJoin);
+      }
       // 5. Add new JoinRel & its RR to the maps
       m_relToHiveColNameOptiqPosMap.put(joinRel,
           this.buildHiveToOptiqColumnMap(joinRR, joinRel));

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q?rev=1619294&r1=1619293&r2=1619294&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/cbo_correctness.q Thu Aug 21 05:40:48 2014
@@ -55,7 +55,6 @@ select t2.c_int+c_float as x , c_int as 
 select t1.c_int, t2.c_int from t1 join             t2 on t1.key=t2.key;
 select t1.key from t1 join t3;
 select t1.key from t1 join t3 where t1.key=t3.key and t1.key >= 1;
---select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key;
 select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key;
 select t1.c_int, t2.c_int from t1 right outer join t2 on t1.key=t2.key;
 select t1.c_int, t2.c_int from t1 full outer join  t2 on t1.key=t2.key;
@@ -77,7 +76,6 @@ select key, t1.c_int, t2.p, q from t1 jo
 
 -- 5. Test Select + Join + FIL + TS
 select t1.c_int, t2.c_int from t1 join t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0);
---select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0);
 select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0);
 select t1.c_int, t2.c_int from t1 right outer join t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0);
 select t1.c_int, t2.c_int from t1 full outer join  t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0);
@@ -95,17 +93,6 @@ select q, b, t2.p, t1.c, t3.c_int from (
 select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 full outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0);
 
 
-
---select * from (select c, b, a from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where  (b + 1 = 2) and (R.b > 0 or c >= 0);
-
---select * from (select t3.c_int, t1.c, b from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 = 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t3.c_int  == 2) and (b > 0 or c_int >= 0)) R where  (R.c_int + 1 = 2) and (R.b > 0 or c_int >= 0);
-
---select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
-
---select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p full outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
-
-
-
 select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0);
 
 select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0);
@@ -136,8 +123,6 @@ select * from (select key, (c_int+c_floa
 
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0) group by c_float, t1.c_int, key order by a) t1 join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key order by q/10 desc, r asc) t2 on t1.a=p join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c order by t3.c_int+c desc, c;
 
---select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc;
 
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b+c, a desc) t1 right outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) t2 on t1.a=p right outer join t3 on t1.a=key where (b + t2.q >= 2) and (b > 0 or c_int >= 0) group by t3.c_int, c;
@@ -155,8 +140,6 @@ select key, c_int from(select key, c_int
 
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0) group by c_float, t1.c_int, key order by a limit 5) t1 join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key order by q/10 desc, r asc limit 5) t2 on t1.a=p join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c order by t3.c_int+c desc, c limit 5;
 
---select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
 select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc limit 5) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 limit 5) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc limit 5;
 
 -- 8. Test UDAF
@@ -222,3 +205,14 @@ select * from t1 union all select * from
 select key from (select key, c_int from (select * from t1 union all select * from t2 where t2.key >=0)r1 union all select key, c_int from t3)r2 where key >=0 order by key;
 select r2.key from (select key, c_int from (select key, c_int from t1 union all select key, c_int from t3 )r1 union all select key, c_int from t3)r2 join   (select key, c_int from (select * from t1 union all select * from t2 where t2.key >=0)r1 union all select key, c_int from t3)r3 on r2.key=r3.key where r3.key >=0 order by r2.key;
 
+-- 12. SemiJoin
+select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key;
+select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0);
+select * from (select c, b, a from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where  (b + 1 = 2) and (R.b > 0 or c >= 0);
+select * from (select t3.c_int, t1.c, b from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 = 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t3.c_int  == 2) and (b > 0 or c_int >= 0)) R where  (R.c_int + 1 = 2) and (R.b > 0 or c_int >= 0);
+select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
+select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p full outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
+select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
+select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
+
+

Modified: hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out?rev=1619294&r1=1619293&r2=1619294&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out Thu Aug 21 05:40:48 2014
@@ -751,7 +751,7 @@ NULL	NULL
 NULL	NULL
 NULL	NULL
 NULL	NULL
-Warning: Shuffle Join JOIN[4][tables = [t1, t3]] in Stage 'Stage-1:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[7][tables = [$hdt$_23, $hdt$_24]] in Stage 'Stage-1:MAPRED' is a cross product
 PREHOOK: query: select t1.key from t1 join t3
 PREHOOK: type: QUERY
 PREHOOK: Input: default@t1
@@ -1260,16 +1260,14 @@ POSTHOOK: Input: default@t3
 1 
 1 
 1 
-PREHOOK: query: --select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key;
-select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key
+PREHOOK: query: select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key
 PREHOOK: type: QUERY
 PREHOOK: Input: default@t1
 PREHOOK: Input: default@t1@dt=2014
 PREHOOK: Input: default@t2
 PREHOOK: Input: default@t2@dt=2014
 #### A masked pattern was here ####
-POSTHOOK: query: --select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key;
-select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key
+POSTHOOK: query: select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@t1
 POSTHOOK: Input: default@t1@dt=2014
@@ -7810,16 +7808,14 @@ POSTHOOK: Input: default@t2@dt=2014
 1	1
 1	1
 1	1
-PREHOOK: query: --select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0);
-select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0)
+PREHOOK: query: select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0)
 PREHOOK: type: QUERY
 PREHOOK: Input: default@t1
 PREHOOK: Input: default@t1@dt=2014
 PREHOOK: Input: default@t2
 PREHOOK: Input: default@t2@dt=2014
 #### A masked pattern was here ####
-POSTHOOK: query: --select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0);
-select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0)
+POSTHOOK: query: select t1.c_int, t2.c_int from t1 left outer join  t2 on t1.key=t2.key where (t1.c_int + t2.c_int == 2) and (t1.c_int > 0 or t2.c_float >= 0)
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@t1
 POSTHOOK: Input: default@t1@dt=2014
@@ -10862,17 +10858,7 @@ POSTHOOK: Input: default@t3
 1	1	1 	1.0	1
 1	1	1 	1.0	1
 1	1	1 	1.0	1
-PREHOOK: query: --select * from (select c, b, a from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where  (b + 1 = 2) and (R.b > 0 or c >= 0);
-
---select * from (select t3.c_int, t1.c, b from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 = 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t3.c_int  == 2) and (b > 0 or c_int >= 0)) R where  (R.c_int + 1 = 2) and (R.b > 0 or c_int >= 0);
-
---select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
-
---select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p full outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
-
-
-
-select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0)
+PREHOOK: query: select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0)
 PREHOOK: type: QUERY
 PREHOOK: Input: default@t1
 PREHOOK: Input: default@t1@dt=2014
@@ -10880,17 +10866,7 @@ PREHOOK: Input: default@t2
 PREHOOK: Input: default@t2@dt=2014
 PREHOOK: Input: default@t3
 #### A masked pattern was here ####
-POSTHOOK: query: --select * from (select c, b, a from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where  (b + 1 = 2) and (R.b > 0 or c >= 0);
-
---select * from (select t3.c_int, t1.c, b from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 = 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t3.c_int  == 2) and (b > 0 or c_int >= 0)) R where  (R.c_int + 1 = 2) and (R.b > 0 or c_int >= 0);
-
---select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
-
---select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p full outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0);
-
-
-
-select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0)
+POSTHOOK: query: select * from (select q, b, t2.p, t1.c, t3.c_int from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left outer join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q == 2) and (b > 0 or c_int >= 0)) R where  (q + 1 = 2) and (R.b > 0 or c_int >= 0)
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@t1
 POSTHOOK: Input: default@t1@dt=2014
@@ -15838,9 +15814,7 @@ POSTHOOK: Input: default@t3
 #### A masked pattern was here ####
 1	12
 1	2
-PREHOOK: query: --select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
-select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc
+PREHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc
 PREHOOK: type: QUERY
 PREHOOK: Input: default@t1
 PREHOOK: Input: default@t1@dt=2014
@@ -15848,9 +15822,7 @@ PREHOOK: Input: default@t2
 PREHOOK: Input: default@t2@dt=2014
 PREHOOK: Input: default@t3
 #### A masked pattern was here ####
-POSTHOOK: query: --select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
-select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc
+POSTHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@t1
 POSTHOOK: Input: default@t1@dt=2014
@@ -15997,9 +15969,7 @@ POSTHOOK: Input: default@t3
 #### A masked pattern was here ####
 1	12
 1	2
-PREHOOK: query: --select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
-select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc limit 5) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 limit 5) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc limit 5
+PREHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc limit 5) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 limit 5) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc limit 5
 PREHOOK: type: QUERY
 PREHOOK: Input: default@t1
 PREHOOK: Input: default@t1@dt=2014
@@ -16007,9 +15977,7 @@ PREHOOK: Input: default@t2
 PREHOOK: Input: default@t2@dt=2014
 PREHOOK: Input: default@t3
 #### A masked pattern was here ####
-POSTHOOK: query: --select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a;
-
-select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc limit 5) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 limit 5) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc limit 5
+POSTHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by b % c asc, b desc limit 5) t1 left outer join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key  having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 limit 5) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t2.q >= 0) and (b > 0 or c_int >= 0) group by t3.c_int, c  having t3.c_int > 0 and (c_int >=1 or c >= 1) and (c_int + c) >= 0  order by t3.c_int % c asc, t3.c_int desc limit 5
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@t1
 POSTHOOK: Input: default@t1@dt=2014
@@ -17439,3 +17407,445 @@ POSTHOOK: Input: default@t3
 3
 3
 3
+PREHOOK: query: -- 12. SemiJoin
+select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: -- 12. SemiJoin
+select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+#### A masked pattern was here ####
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+NULL
+NULL
+PREHOOK: query: select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select t1.c_int           from t1 left semi join   t2 on t1.key=t2.key where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+#### A masked pattern was here ####
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+PREHOOK: query: select * from (select c, b, a from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where  (b + 1 = 2) and (R.b > 0 or c >= 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select c, b, a from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c >= 0)) R where  (b + 1 = 2) and (R.b > 0 or c >= 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+1.0	1	 1
+1.0	1	 1
+1.0	1	 1 
+1.0	1	 1 
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1
+1.0	1	1 
+1.0	1	1 
+PREHOOK: query: select * from (select t3.c_int, t1.c, b from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 = 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t3.c_int  == 2) and (b > 0 or c_int >= 0)) R where  (R.c_int + 1 = 2) and (R.b > 0 or c_int >= 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select t3.c_int, t1.c, b from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 = 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p left outer join t3 on t1.a=key where (b + t3.c_int  == 2) and (b > 0 or c_int >= 0)) R where  (R.c_int + 1 = 2) and (R.b > 0 or c_int >= 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+1	1.0	1
+PREHOOK: query: select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p right outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+PREHOOK: query: select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p full outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select c_int, b, t1.c from (select key as a, c_int as b, t1.c_float as c from t1  where (t1.c_int + 1 == 2) and (t1.c_int > 0 or t1.c_float >= 0)) t1 left semi join (select t2.key as p, t2.c_int as q, c_float as r from t2  where (t2.c_int + 1 == 2) and (t2.c_int > 0 or t2.c_float >= 0)) t2 on t1.a=p full outer join t3 on t1.a=key where (b + 1 == 2) and (b > 0 or c_int >= 0)) R where  (c + 1 = 2) and (R.b > 0 or c_int >= 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+1	1	1.0
+PREHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+ 1	2
+ 1 	2
+1 	2
+1	12
+PREHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+PREHOOK: Input: default@t1@dt=2014
+PREHOOK: Input: default@t2
+PREHOOK: Input: default@t2@dt=2014
+PREHOOK: Input: default@t3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from (select key as a, c_int+1 as b, sum(c_int) as c from t1 where (t1.c_int + 1 >= 0) and (t1.c_int > 0 or t1.c_float >= 0)  group by c_float, t1.c_int, key having t1.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by a+b desc, c asc limit 5) t1 left semi join (select key as p, c_int+1 as q, sum(c_int) as r from t2 where (t2.c_int + 1 >= 0) and (t2.c_int > 0 or t2.c_float >= 0)  group by c_float, t2.c_int, key having t2.c_float > 0 and (c_int >=1 or c_float >= 1) and (c_int + c_float) >= 0 order by q+r/10 desc, p limit 5) t2 on t1.a=p left semi join t3 on t1.a=key where (b + 1  >= 0) and (b > 0 or a >= 0) group by a, c  having a > 0 and (a >=1 or c >= 1) and (a + c) >= 0 order by c, a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+POSTHOOK: Input: default@t1@dt=2014
+POSTHOOK: Input: default@t2
+POSTHOOK: Input: default@t2@dt=2014
+POSTHOOK: Input: default@t3
+#### A masked pattern was here ####
+ 1	2
+ 1 	2
+1 	2
+1	12



Mime
View raw message