pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From roh...@apache.org
Subject svn commit: r1426346 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/relational/LOSort.java src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java test/e2e/pig/tests/nightly.conf
Date Thu, 27 Dec 2012 22:11:28 GMT
Author: rohini
Date: Thu Dec 27 22:11:27 2012
New Revision: 1426346

URL: http://svn.apache.org/viewvc?rev=1426346&view=rev
Log:
PIG-3051: java.lang.IndexOutOfBoundsException failure with LimitOptimizer + ColumnPruning
(knoguchi via rohini)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java
    pig/trunk/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java
    pig/trunk/test/e2e/pig/tests/nightly.conf

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1426346&r1=1426345&r2=1426346&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Dec 27 22:11:27 2012
@@ -404,6 +404,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-3051: java.lang.IndexOutOfBoundsException failure with LimitOptimizer + ColumnPruning
(knoguchi via rohini)
+
 PIG-3076: make TestScalarAliases more reliable (julien)
 
 PIG-3020: "Duplicate uid in schema" error when joining two relations derived from the same
load statement (jcoveney)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java?rev=1426346&r1=1426345&r2=1426346&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOSort.java Thu Dec 27 22:11:27
2012
@@ -29,7 +29,9 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.newplan.Operator;
 import org.apache.pig.newplan.OperatorPlan;
 import org.apache.pig.newplan.PlanVisitor;
+import org.apache.pig.newplan.ReverseDependencyOrderWalker;
 import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
 import org.apache.pig.newplan.logical.expression.ProjectExpression;
 
 public class LOSort extends LogicalRelationalOperator{
@@ -197,4 +199,42 @@ public class LOSort extends LogicalRelat
     public Operator getInput(LogicalPlan plan) {
         return plan.getPredecessors(this).get(0);
     }
+
+    private static class ResetProjectionAttachedRelationalOpVisitor
+        extends LogicalExpressionVisitor {
+        private LogicalRelationalOperator attachedRelationalOp;
+
+        ResetProjectionAttachedRelationalOpVisitor (
+            LogicalExpressionPlan plan, LogicalRelationalOperator op )
+            throws FrontendException {
+            super(plan, new ReverseDependencyOrderWalker(plan));
+            this.attachedRelationalOp = op;
+
+        }
+        @Override
+        public void visit(ProjectExpression pe) throws FrontendException {
+            pe.setAttachedRelationalOp(attachedRelationalOp);
+        }
+    }
+
+    public static LOSort createCopy(LOSort sort) throws FrontendException {
+        LOSort newSort = new LOSort(sort.getPlan(), null,
+                                    sort.getAscendingCols(),
+                                    sort.getUserFunc());
+
+        List<LogicalExpressionPlan> newSortColPlans =
+            new ArrayList<LogicalExpressionPlan>(sort.getSortColPlans().size());
+
+        for(LogicalExpressionPlan lep:sort.getSortColPlans() ) {
+            LogicalExpressionPlan new_lep = lep.deepCopy();
+
+            // Resetting the attached LOSort operator of the ProjectExpression
+            // to the newSort 
+            new ResetProjectionAttachedRelationalOpVisitor(
+                new_lep, newSort ).visit();
+            newSortColPlans.add(new_lep);
+        }
+        newSort.setSortColPlans(newSortColPlans);
+        return newSort;
+    }
 }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java?rev=1426346&r1=1426345&r2=1426346&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java Thu Dec 27 22:11:27
2012
@@ -202,9 +202,7 @@ public class LimitOptimizer extends Rule
                             && greatGrandparants.size() != 0
                             && greatGrandparants.get(0) instanceof LOSort) {
                         LOSort sort = (LOSort) greatGrandparants.get(0);
-                        LOSort newSort = new LOSort(sort.getPlan(), sort
-                                .getSortColPlans(), sort.getAscendingCols(),
-                                sort.getUserFunc());
+                        LOSort newSort = LOSort.createCopy(sort);
                         newSort.setLimit(limit.getLimit());
 
                         currentPlan.replace(limit, newSort);

Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1426346&r1=1426345&r2=1426346&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Thu Dec 27 22:11:27 2012
@@ -4575,6 +4575,27 @@ store C into ':OUTPATH:';\, 
                         'rc' => 0,
                         'not_expected_out_regex' => "checkJobConf: conf is null: false",
                         'expected_out_regex' => "checkJobConf: conf is null: true",
+                    }, {
+                        # PIG-3051
+                        'num' => 5,
+                        'pig' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name,
age, registration, contributions);
+                                -- dropping one column to force columnprune
+                                B = foreach A generate age, registration,contributions;
+                                C = order B by contributions;
+                                D = limit C 3;
+                                E = foreach D generate contributions;
+                                STORE C INTO ':OUTPATH:.1';
+                                STORE E INTO ':OUTPATH:.2';?,
+                        'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/votertab10k'
AS (name, age, registration, contributions);
+                                B = foreach A generate age, registration,contributions;
+                                C = order B by contributions;
+                                STORE C INTO ':OUTPATH:.1';
+                                F = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age,
registration, contributions);
+                                G = foreach F generate age, registration,contributions;
+                                H = order G by contributions;
+                                I = limit H 3;
+                                J = foreach I generate contributions;
+                                STORE J INTO ':OUTPATH:.2';?,
                     }
                 ],
             },{



Mime
View raw message