Return-Path: Delivered-To: apmail-pig-commits-archive@www.apache.org Received: (qmail 95174 invoked from network); 11 Dec 2010 03:59:54 -0000 Received: from unknown (HELO mail.apache.org) (140.211.11.3) by 140.211.11.9 with SMTP; 11 Dec 2010 03:59:54 -0000 Received: (qmail 12048 invoked by uid 500); 11 Dec 2010 03:59:54 -0000 Delivered-To: apmail-pig-commits-archive@pig.apache.org Received: (qmail 11989 invoked by uid 500); 11 Dec 2010 03:59:53 -0000 Mailing-List: contact commits-help@pig.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pig.apache.org Delivered-To: mailing list commits@pig.apache.org Received: (qmail 11982 invoked by uid 99); 11 Dec 2010 03:59:53 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 11 Dec 2010 03:59:53 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 11 Dec 2010 03:59:50 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 74F3323889E1; Sat, 11 Dec 2010 03:59:28 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1044564 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java test/org/apache/pig/test/TestFilterSimplification.java Date: Sat, 11 Dec 2010 03:59:28 -0000 To: commits@pig.apache.org From: yanz@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20101211035928.74F3323889E1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: yanz Date: Sat Dec 11 03:59:28 2010 New Revision: 1044564 URL: http://svn.apache.org/viewvc?rev=1044564&view=rev Log: PIG-1762: Logical simplification fails on map key referenced values (yanz) Modified: pig/trunk/CHANGES.txt pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1044564&r1=1044563&r2=1044564&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Sat Dec 11 03:59:28 2010 @@ -234,6 +234,8 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1762: Logical simplification fails on map key referenced values (yanz) + PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy) PIG-1757: After split combination, the number of maps may vary slightly (yanz) Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java?rev=1044564&r1=1044563&r2=1044564&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java (original) +++ pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java Sat Dec 11 03:59:28 2010 @@ -58,8 +58,22 @@ public class MapLookupExpression extends public boolean isEqual(Operator other) throws FrontendException { if (other != null && other instanceof MapLookupExpression) { MapLookupExpression po = (MapLookupExpression)other; - return ( po.mMapKey.compareTo(mMapKey) == 0 ) && - po.mValueSchema.isEqual( mValueSchema ); + if ( po.mMapKey.compareTo(mMapKey) != 0 || + !po.mValueSchema.isEqual( mValueSchema )) + return false; + else { + // check the nested map equality + if (plan.getSuccessors(this) != null) { + if (other.getPlan().getSuccessors(other) == null) + return false; + else { + return plan.getSuccessors(this).get(0).isEqual(other.getPlan().getSuccessors(other).get(0)); + } + } else if (other.getPlan().getSuccessors(other) != null) { + return false; + } else + return true; + } } else { return false; } Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java?rev=1044564&r1=1044563&r2=1044564&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java (original) +++ pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java Sat Dec 11 03:59:28 2010 @@ -33,7 +33,7 @@ import org.apache.pig.newplan.OperatorPl * generated. * */ -class DNFPlanGenerator extends AllSameExpressionVisitor { +class DNFPlanGenerator extends LogicalExpressionVisitor { private OperatorPlan dnfPlan = null; Stack result; @@ -42,34 +42,15 @@ class DNFPlanGenerator extends AllSameEx result = new Stack(); } - @Override - protected void execute(LogicalExpression op) - throws FrontendException { - if (op instanceof UnaryExpression) { - result.pop(); - } - else if (op instanceof BinaryExpression) { - result.pop(); - result.pop(); - } - else if (op instanceof BinCondExpression) { - result.pop(); - result.pop(); - result.pop(); - } - - result.push(op); - } - OperatorPlan getDNFPlan() { - if (dnfPlan == null) dnfPlan = result.pop().getPlan(); + if (dnfPlan == null) dnfPlan = (result.empty() ? plan : result.pop().getPlan()); return dnfPlan; } @Override public void visit(AndExpression exp) throws FrontendException { - LogicalExpression rhsExp = result.pop(); - LogicalExpression lhsExp = result.pop(); + LogicalExpression rhsExp = ((exp.getRhs() instanceof AndExpression || exp.getRhs() instanceof OrExpression ? result.pop() : exp.getRhs())); + LogicalExpression lhsExp = ((exp.getLhs() instanceof AndExpression || exp.getLhs() instanceof OrExpression ? result.pop() : exp.getLhs())); if (!(lhsExp instanceof AndExpression) && !(lhsExp instanceof DNFExpression) && !(lhsExp instanceof OrExpression) && !(rhsExp instanceof AndExpression) && !(rhsExp instanceof OrExpression) && !(rhsExp instanceof DNFExpression)) result.push(exp); else { if (dnfPlan == null) dnfPlan = new DNFPlan(); @@ -368,8 +349,8 @@ class DNFPlanGenerator extends AllSameEx @Override public void visit(OrExpression exp) throws FrontendException { - LogicalExpression rhsExp = result.pop(); - LogicalExpression lhsExp = result.pop(); + LogicalExpression rhsExp = ((exp.getRhs() instanceof AndExpression || exp.getRhs() instanceof OrExpression ? result.pop() : exp.getRhs())); + LogicalExpression lhsExp = ((exp.getLhs() instanceof AndExpression || exp.getLhs() instanceof OrExpression ? result.pop() : exp.getLhs())); if (!(lhsExp instanceof OrExpression) && (!(lhsExp instanceof DNFExpression) || ((DNFExpression) lhsExp).type == DNFExpression.DNFExpressionType.AND) && !(rhsExp instanceof OrExpression) && (!(rhsExp instanceof DNFExpression) || ((DNFExpression) rhsExp).type == DNFExpression.DNFExpressionType.AND)) result.push(exp); Modified: pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java?rev=1044564&r1=1044563&r2=1044564&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java (original) +++ pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java Sat Dec 11 03:59:28 2010 @@ -770,6 +770,40 @@ public class TestFilterSimplification ex assertTrue(expected.isEqual(newLogicalPlan)); } + @Test + public void test7() throws Exception { + LogicalPlanTester lpt = new LogicalPlanTester(pc); + lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);"); + + org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';"); + LogicalPlan newLogicalPlan = migratePlan(plan); + + PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10); + optimizer.optimize(); + + lpt = new LogicalPlanTester(pc); + lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);"); + plan = lpt.buildPlan("store b into 'empty';"); + LogicalPlan expected = migratePlan(plan); + + assertTrue(expected.isEqual(newLogicalPlan)); + + lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);"); + + plan = lpt.buildPlan("store b into 'empty';"); + newLogicalPlan = migratePlan(plan); + + optimizer = new MyPlanOptimizer(newLogicalPlan, 10); + optimizer.optimize(); + + lpt = new LogicalPlanTester(pc); + lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f' is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);"); + plan = lpt.buildPlan("store b into 'empty';"); + expected = migratePlan(plan); + + assertTrue(expected.isEqual(newLogicalPlan)); + } + public class MyPlanOptimizer extends LogicalPlanOptimizer { protected MyPlanOptimizer(OperatorPlan p, int iterations) {