hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1459379 - in /hive/branches/ptf-windowing/ql/src: java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java test/queries/clientpositive/windowing_rank.q test/results/clientpositive/windowing_rank.q.out
Date Thu, 21 Mar 2013 16:10:41 GMT
Author: hashutosh
Date: Thu Mar 21 16:10:40 2013
New Revision: 1459379

URL: http://svn.apache.org/r1459379
Log:
HIVE-4180

Summary: Filter getting pushed above PTF Operator when it should not.

Test Plan: Added new test which demonstrates the problem.

Reviewers: hbutani

Differential Revision: https://reviews.facebook.net/D9435

Modified:
    hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
    hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_rank.q
    hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_rank.q.out

Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1459379&r1=1459378&r2=1459379&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
(original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
Thu Mar 21 16:10:40 2013
@@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.exec.Fi
 import org.apache.hadoop.hive.ql.exec.JoinOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
-import org.apache.hadoop.hive.ql.exec.PTFOperator;
 import org.apache.hadoop.hive.ql.exec.RowSchema;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -532,32 +531,10 @@ public final class OpProcFactory {
     public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
         Object... nodeOutputs) throws SemanticException {
       LOG.info("Processing for " + nd.getName() + "("+ ((Operator<?>) nd).getIdentifier()
+ ")");
-      OpWalkerInfo owi = (OpWalkerInfo) procCtx;
-      PTFOperator ptfOp = (PTFOperator)nd;
-
-      if (ptfOp.getConf().getFuncDef().getOrder().getExpressions() != null) {
-        // If there is an ordering, it is not safe to push up predicate e.g., rank, lead,
lag
-        // However currently if user hasn't specified any ordering, we order on partitioning
-        //  column so we will always hit this if condition. In future, if we relax this condition
-        // than we can push up filter (if it satisfies other conditions).
-        return null;
-      }
-      // We also can't push filters if filter is on column generated by PTFOp.
-      // To deny those cases, check if filter expression references any such column. These
-      // columns can be identified since their internal name begins with _wcol.
-      // That check hasn't been implemented yet because we will not come here until previous
-      // comment is addressed.
-      boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, null, false);
-      if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
-          HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
-        if (hasUnpushedPredicates) {
-          ExprWalkerInfo unpushedPreds = mergeChildrenPred(nd, owi, null, false);
-          return createFilter(ptfOp, unpushedPreds, owi);
-        }
-      }
+      // It's not safe to push filters up through PTFPPD.
+      ((OpWalkerInfo) procCtx).getCandidateFilterOps().clear();
       return null;
     }
-
   }
   /**
    * Default processor which just merges its children.

Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_rank.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_rank.q?rev=1459379&r1=1459378&r2=1459379&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_rank.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/windowing_rank.q Thu Mar
21 16:10:40 2013
@@ -51,3 +51,16 @@ from
   ) ranked
 where dec = 89.5 limit 10;
 
+select ts, dec, rnk
+from
+  (select ts, dec,
+          rank() over (partition by ts)  as rnk
+          from
+            (select other.ts, other.dec
+             from over10k other
+             join over10k on (other.b = over10k.b)
+             where other.t < 10
+            ) joined
+  ) ranked
+where rnk = 1 limit 10;
+

Modified: hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_rank.q.out
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_rank.q.out?rev=1459379&r1=1459378&r2=1459379&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_rank.q.out (original)
+++ hive/branches/ptf-windowing/ql/src/test/results/clientpositive/windowing_rank.q.out Thu
Mar 21 16:10:40 2013
@@ -551,3 +551,43 @@ POSTHOOK: Input: default@over10k
 2013-03-01 09:11:58.136070312	89.5	1
 2013-03-01 09:11:58.136070312	89.5	1
 2013-03-01 09:11:58.136070312	89.5	1
+PREHOOK: query: select ts, dec, rnk
+from
+  (select ts, dec,
+          rank() over (partition by ts)  as rnk
+          from
+            (select other.ts, other.dec
+             from over10k other
+             join over10k on (other.b = over10k.b)
+             where other.t < 10
+            ) joined
+  ) ranked
+where rnk = 1 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, dec, rnk
+from
+  (select ts, dec,
+          rank() over (partition by ts)  as rnk
+          from
+            (select other.ts, other.dec
+             from over10k other
+             join over10k on (other.b = over10k.b)
+             where other.t < 10
+            ) joined
+  ) ranked
+where rnk = 1 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1
+2013-03-01 09:11:58.136070307	52.01	1



Mime
View raw message