pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1791451 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/rules/NestedLimitOptimizer.java test/org/apache/pig/test/TestOptimizeNestedLimit.java test/org/apache/pig/test/TestSecondarySort.java
Date Sat, 15 Apr 2017 05:31:02 GMT
Author: daijy
Date: Sat Apr 15 05:31:01 2017
New Revision: 1791451

URL: http://svn.apache.org/viewvc?rev=1791451&view=rev
Log:
PIG-5223: TestLimitVariable.testNestedLimitVariable1 and TestSecondarySortMR.testNestedLimitedSort
failing

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/rules/NestedLimitOptimizer.java
    pig/trunk/test/org/apache/pig/test/TestOptimizeNestedLimit.java
    pig/trunk/test/org/apache/pig/test/TestSecondarySort.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1791451&r1=1791450&r2=1791451&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Apr 15 05:31:01 2017
@@ -97,6 +97,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5223: TestLimitVariable.testNestedLimitVariable1 and TestSecondarySortMR.testNestedLimitedSort
failing (jins via daijy)
+
 PIG-5209: Cross product on flatten(map) fails with ClassCastException (knoguchi)
 
 PIG-5153: Change of behavior in FLATTEN(map) (szita via rohini)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/NestedLimitOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/NestedLimitOptimizer.java?rev=1791451&r1=1791450&r2=1791451&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/NestedLimitOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/NestedLimitOptimizer.java Sat Apr 15
05:31:01 2017
@@ -65,8 +65,11 @@ public class NestedLimitOptimizer extend
                 Operator op = it.next();
                 if (op instanceof LOLimit) {
                     List<Operator> preds = innerPlan.getPredecessors(op);
+                    LOLimit limitOp = (LOLimit) op;
+                    boolean useExpressionPlan = limitOp.getLimit() == -1 &&
+                            limitOp.getLimitPlan() != null;
                     // Limit should always have exactly 1 predecessor
-                    if (preds.get(0) instanceof LOSort) {
+                    if (!useExpressionPlan && preds.get(0) instanceof LOSort) {
                         return true;
                     }
                 }
@@ -94,8 +97,11 @@ public class NestedLimitOptimizer extend
                 Operator op = it.next();
                 if (op instanceof LOLimit) {
                     List<Operator> preds = innerPlan.getPredecessors(op);
+                    LOLimit limitOp = (LOLimit) op;
+                    boolean useExpressionPlan = limitOp.getLimit() == -1 &&
+                            limitOp.getLimitPlan() != null;
                     // Limit should always have exactly 1 predecessor
-                    if (preds.get(0) instanceof LOSort) {
+                    if (!useExpressionPlan && preds.get(0) instanceof LOSort) {
                         limit = (LOLimit) op;
                         sort = (LOSort) (preds.get(0));
                         break;

Modified: pig/trunk/test/org/apache/pig/test/TestOptimizeNestedLimit.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestOptimizeNestedLimit.java?rev=1791451&r1=1791450&r2=1791451&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestOptimizeNestedLimit.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestOptimizeNestedLimit.java Sat Apr 15 05:31:01 2017
@@ -58,6 +58,7 @@ import org.junit.Test;
 public class TestOptimizeNestedLimit {
 
     LogicalPlan lp;
+    LogicalPlan lp1;
     PhysicalPlan php;
     MROperPlan mrp;
 
@@ -76,6 +77,18 @@ public class TestOptimizeNestedLimit {
         lp = optimizePlan(Util.buildLp(pigServer, query));
         php = Util.buildPp(pigServer, query);
         mrp = Util.buildMRPlanWithOptimizer(php, pc);
+
+        query = "a = load 'myfile' as (id:int, num:int);" +
+                "b = group a by num;" +
+                "c = foreach b generate COUNT(a) as ntup;" +
+                "d = group c all;" +
+                "e = foreach d generate MIN(c.ntup) AS min;" +
+                "f = foreach b {" +
+                "g = order a by id ASC;" +
+                "h = limit g e.min;" +
+                "generate FLATTEN(h);}" +
+                "store f into 'empty';";
+        lp1 = optimizePlan(Util.buildLp(pigServer, query));
     }
 
     @After
@@ -115,6 +128,43 @@ public class TestOptimizeNestedLimit {
     }
 
     @Test
+    // Test logical plan not being optimized with LOLimit.mLimit = -1
+    public void testLogicalPlan1() throws Exception {
+
+        Iterator<Operator> it = lp1.getOperators();
+
+        LOForEach forEach = null;
+        LOSort sort = null;
+        LOLimit limit = null;
+
+        while(it.hasNext()) {
+            Operator op = it.next();
+            if (op instanceof LOForEach) {
+                forEach = (LOForEach) op;
+
+                Iterator<Operator> innerIt = forEach.getInnerPlan().getOperators();
+
+                while(innerIt.hasNext()) {
+                    Operator innerOp = innerIt.next();
+                    if (innerOp instanceof LOSort) {
+                        assertNull("There should be only one LOSort", sort);
+                        sort = (LOSort) innerOp;
+                    } else if (innerOp instanceof LOLimit) {
+                        assertNull("There should be only one LOLimit", limit);
+                        limit = (LOLimit) innerOp;
+                    }
+                }
+            }
+        }
+
+        assertNotNull("ForEach is missing", forEach);
+        assertNotNull("LOSort is missing", sort);
+        assertNotNull("LOLimit is missing", limit);
+        assertEquals(sort.getLimit(), -1);
+        assertEquals(limit.getLimit(), -1);
+    }
+
+    @Test
     // test physical plan
     public void testPhysicalPlan() throws Exception {
 

Modified: pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=1791451&r1=1791450&r2=1791451&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Sat Apr 15 05:31:01 2017
@@ -552,7 +552,7 @@ public abstract class TestSecondarySort
                        "c1 = order a by age;" +
                        "c2 = limit c1 5;" +
                        "generate c2;}" +
-                       "store c in 'empty';";
+                       "store c into 'empty';";
 
         SecondaryKeyOptimizer so = visitSecondaryKeyOptimizer(query);
 



Mime
View raw message