pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From y...@apache.org
Subject svn commit: r1044564 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java test/org/apache/pig/test/TestFilterSimplification.java
Date Sat, 11 Dec 2010 03:59:28 GMT
Author: yanz
Date: Sat Dec 11 03:59:28 2010
New Revision: 1044564

URL: http://svn.apache.org/viewvc?rev=1044564&view=rev
Log:
PIG-1762: Logical simplification fails on map key referenced values (yanz)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java
    pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java
    pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1044564&r1=1044563&r2=1044564&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Dec 11 03:59:28 2010
@@ -234,6 +234,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1762: Logical simplification fails on map key referenced values (yanz)
+
 PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
 
 PIG-1757: After split combination, the number of maps may vary slightly (yanz)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java?rev=1044564&r1=1044563&r2=1044564&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/MapLookupExpression.java Sat Dec
11 03:59:28 2010
@@ -58,8 +58,22 @@ public class MapLookupExpression extends
     public boolean isEqual(Operator other) throws FrontendException {
         if (other != null && other instanceof MapLookupExpression) {
             MapLookupExpression po = (MapLookupExpression)other;
-            return ( po.mMapKey.compareTo(mMapKey) == 0 ) && 
-            po.mValueSchema.isEqual( mValueSchema );
+            if ( po.mMapKey.compareTo(mMapKey) != 0  || 
+                 !po.mValueSchema.isEqual( mValueSchema ))
+                return false;
+            else {
+                // check the nested map equality
+                if (plan.getSuccessors(this) != null) {
+                    if (other.getPlan().getSuccessors(other) == null)
+                        return false;
+                    else {
+                        return plan.getSuccessors(this).get(0).isEqual(other.getPlan().getSuccessors(other).get(0));
+                    }
+                } else if (other.getPlan().getSuccessors(other) != null) {
+                    return false;
+                } else
+                    return true;
+            }
         } else {
             return false;
         }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java?rev=1044564&r1=1044563&r2=1044564&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/DNFPlanGenerator.java Sat Dec 11 03:59:28
2010
@@ -33,7 +33,7 @@ import org.apache.pig.newplan.OperatorPl
  * generated.
  *
  */
-class DNFPlanGenerator extends AllSameExpressionVisitor {
+class DNFPlanGenerator extends LogicalExpressionVisitor {
     private OperatorPlan dnfPlan = null;
     Stack<LogicalExpression> result;
 
@@ -42,34 +42,15 @@ class DNFPlanGenerator extends AllSameEx
         result = new Stack<LogicalExpression>();
     }
 
-    @Override
-    protected void execute(LogicalExpression op)
-                    throws FrontendException {
-        if (op instanceof UnaryExpression) {
-            result.pop();
-        }
-        else if (op instanceof BinaryExpression) {
-            result.pop();
-            result.pop();
-        }
-        else if (op instanceof BinCondExpression) {
-            result.pop();
-            result.pop();
-            result.pop();
-        }
-
-        result.push(op);
-    }
-
     OperatorPlan getDNFPlan() {
-        if (dnfPlan == null) dnfPlan = result.pop().getPlan();
+        if (dnfPlan == null) dnfPlan = (result.empty() ? plan : result.pop().getPlan());
         return dnfPlan;
     }
 
     @Override
     public void visit(AndExpression exp) throws FrontendException {
-        LogicalExpression rhsExp = result.pop();
-        LogicalExpression lhsExp = result.pop();
+        LogicalExpression rhsExp = ((exp.getRhs() instanceof AndExpression || exp.getRhs()
instanceof OrExpression ? result.pop() : exp.getRhs()));
+        LogicalExpression lhsExp = ((exp.getLhs() instanceof AndExpression || exp.getLhs()
instanceof OrExpression ? result.pop() : exp.getLhs()));
         if (!(lhsExp instanceof AndExpression) && !(lhsExp instanceof DNFExpression)
&& !(lhsExp instanceof OrExpression) && !(rhsExp instanceof AndExpression)
&& !(rhsExp instanceof OrExpression) && !(rhsExp instanceof DNFExpression))
result.push(exp);
         else {
             if (dnfPlan == null) dnfPlan = new DNFPlan();
@@ -368,8 +349,8 @@ class DNFPlanGenerator extends AllSameEx
 
     @Override
     public void visit(OrExpression exp) throws FrontendException {
-        LogicalExpression rhsExp = result.pop();
-        LogicalExpression lhsExp = result.pop();
+        LogicalExpression rhsExp = ((exp.getRhs() instanceof AndExpression || exp.getRhs()
instanceof OrExpression ? result.pop() : exp.getRhs()));
+        LogicalExpression lhsExp = ((exp.getLhs() instanceof AndExpression || exp.getLhs()
instanceof OrExpression ? result.pop() : exp.getLhs()));
         if (!(lhsExp instanceof OrExpression) && 
             (!(lhsExp instanceof DNFExpression) || 
                 ((DNFExpression) lhsExp).type == DNFExpression.DNFExpressionType.AND) &&
!(rhsExp instanceof OrExpression) && (!(rhsExp instanceof DNFExpression) || ((DNFExpression)
rhsExp).type == DNFExpression.DNFExpressionType.AND)) result.push(exp);

Modified: pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java?rev=1044564&r1=1044563&r2=1044564&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestFilterSimplification.java Sat Dec 11 03:59:28 2010
@@ -770,6 +770,40 @@ public class TestFilterSimplification ex
         assertTrue(expected.isEqual(newLogicalPlan));
     }
 
+    @Test
+    public void test7() throws Exception {
+        LogicalPlanTester lpt = new LogicalPlanTester(pc);
+        lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f'
is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);"); 
+
+        org.apache.pig.impl.logicalLayer.LogicalPlan plan = lpt.buildPlan("store b into 'empty';");
+        LogicalPlan newLogicalPlan = migratePlan(plan);
+
+        PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
+        optimizer.optimize();
+
+        lpt = new LogicalPlanTester(pc);
+        lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f'
is not null and (v2#'f'#'f1' is not null or v2#'f'#'f2' is not null);"); 
+        plan = lpt.buildPlan("store b into 'empty';");
+        LogicalPlan expected = migratePlan(plan);
+
+        assertTrue(expected.isEqual(newLogicalPlan));
+        
+        lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f'
is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);"); 
+
+        plan = lpt.buildPlan("store b into 'empty';");
+        newLogicalPlan = migratePlan(plan);
+
+        optimizer = new MyPlanOptimizer(newLogicalPlan, 10);
+        optimizer.optimize();
+
+        lpt = new LogicalPlanTester(pc);
+        lpt.buildPlan("b = filter (load 'd.txt' as (k1, k2, k3, v1, v2, v3)) by k2#'f1'#'f'
is not null and (v2#'f1'#'f' is not null or v2#'f2'#'f' is not null);"); 
+        plan = lpt.buildPlan("store b into 'empty';");
+        expected = migratePlan(plan);
+
+        assertTrue(expected.isEqual(newLogicalPlan));
+    }
+    
     public class MyPlanOptimizer extends LogicalPlanOptimizer {
 
         protected MyPlanOptimizer(OperatorPlan p, int iterations) {



Mime
View raw message