Return-Path: X-Original-To: apmail-pig-commits-archive@www.apache.org Delivered-To: apmail-pig-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 95741DBF2 for ; Thu, 27 Dec 2012 22:11:24 +0000 (UTC) Received: (qmail 11449 invoked by uid 500); 27 Dec 2012 22:11:24 -0000 Delivered-To: apmail-pig-commits-archive@pig.apache.org Received: (qmail 11371 invoked by uid 500); 27 Dec 2012 22:11:24 -0000 Mailing-List: contact commits-help@pig.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pig.apache.org Delivered-To: mailing list commits@pig.apache.org Received: (qmail 11362 invoked by uid 99); 27 Dec 2012 22:11:24 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 27 Dec 2012 22:11:24 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 27 Dec 2012 22:11:23 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 013A423889EA; Thu, 27 Dec 2012 22:11:02 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1426345 - in /pig/branches/branch-0.11: CHANGES.txt src/org/apache/pig/newplan/logical/relational/LOSort.java src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java test/e2e/pig/tests/nightly.conf Date: Thu, 27 Dec 2012 22:11:01 -0000 To: commits@pig.apache.org From: rohini@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121227221102.013A423889EA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: rohini Date: Thu Dec 27 22:11:01 2012 New Revision: 1426345 URL: http://svn.apache.org/viewvc?rev=1426345&view=rev Log: PIG-3051: java.lang.IndexOutOfBoundsException failure with LimitOptimizer + ColumnPruning (knoguchi via rohini) Modified: pig/branches/branch-0.11/CHANGES.txt pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf Modified: pig/branches/branch-0.11/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/CHANGES.txt?rev=1426345&r1=1426344&r2=1426345&view=diff ============================================================================== --- pig/branches/branch-0.11/CHANGES.txt (original) +++ pig/branches/branch-0.11/CHANGES.txt Thu Dec 27 22:11:01 2012 @@ -332,6 +332,8 @@ OPTIMIZATIONS BUG FIXES +PIG-3051: java.lang.IndexOutOfBoundsException failure with LimitOptimizer + ColumnPruning (knoguchi via rohini) + PIG-3076: make TestScalarAliases more reliable (julien) PIG-3020: "Duplicate uid in schema" error when joining two relations derived from the same load statement (jcoveney) Modified: pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java?rev=1426345&r1=1426344&r2=1426345&view=diff ============================================================================== --- pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java (original) +++ pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/relational/LOSort.java Thu Dec 27 22:11:01 2012 @@ -29,7 +29,9 @@ import org.apache.pig.impl.logicalLayer. import org.apache.pig.newplan.Operator; import org.apache.pig.newplan.OperatorPlan; import org.apache.pig.newplan.PlanVisitor; +import org.apache.pig.newplan.ReverseDependencyOrderWalker; import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan; +import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor; import org.apache.pig.newplan.logical.expression.ProjectExpression; public class LOSort extends LogicalRelationalOperator{ @@ -197,4 +199,42 @@ public class LOSort extends LogicalRelat public Operator getInput(LogicalPlan plan) { return plan.getPredecessors(this).get(0); } + + private static class ResetProjectionAttachedRelationalOpVisitor + extends LogicalExpressionVisitor { + private LogicalRelationalOperator attachedRelationalOp; + + ResetProjectionAttachedRelationalOpVisitor ( + LogicalExpressionPlan plan, LogicalRelationalOperator op ) + throws FrontendException { + super(plan, new ReverseDependencyOrderWalker(plan)); + this.attachedRelationalOp = op; + + } + @Override + public void visit(ProjectExpression pe) throws FrontendException { + pe.setAttachedRelationalOp(attachedRelationalOp); + } + } + + public static LOSort createCopy(LOSort sort) throws FrontendException { + LOSort newSort = new LOSort(sort.getPlan(), null, + sort.getAscendingCols(), + sort.getUserFunc()); + + List newSortColPlans = + new ArrayList(sort.getSortColPlans().size()); + + for(LogicalExpressionPlan lep:sort.getSortColPlans() ) { + LogicalExpressionPlan new_lep = lep.deepCopy(); + + // Resetting the attached LOSort operator of the ProjectExpression + // to the newSort + new ResetProjectionAttachedRelationalOpVisitor( + new_lep, newSort ).visit(); + newSortColPlans.add(new_lep); + } + newSort.setSortColPlans(newSortColPlans); + return newSort; + } } Modified: pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java?rev=1426345&r1=1426344&r2=1426345&view=diff ============================================================================== --- pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java (original) +++ pig/branches/branch-0.11/src/org/apache/pig/newplan/logical/rules/LimitOptimizer.java Thu Dec 27 22:11:01 2012 @@ -202,9 +202,7 @@ public class LimitOptimizer extends Rule && greatGrandparants.size() != 0 && greatGrandparants.get(0) instanceof LOSort) { LOSort sort = (LOSort) greatGrandparants.get(0); - LOSort newSort = new LOSort(sort.getPlan(), sort - .getSortColPlans(), sort.getAscendingCols(), - sort.getUserFunc()); + LOSort newSort = LOSort.createCopy(sort); newSort.setLimit(limit.getLimit()); currentPlan.replace(limit, newSort); Modified: pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf?rev=1426345&r1=1426344&r2=1426345&view=diff ============================================================================== --- pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf (original) +++ pig/branches/branch-0.11/test/e2e/pig/tests/nightly.conf Thu Dec 27 22:11:01 2012 @@ -4575,6 +4575,27 @@ store C into ':OUTPATH:';\, 'rc' => 0, 'not_expected_out_regex' => "checkJobConf: conf is null: false", 'expected_out_regex' => "checkJobConf: conf is null: true", + }, { + # PIG-3051 + 'num' => 5, + 'pig' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions); + -- dropping one column to force columnprune + B = foreach A generate age, registration,contributions; + C = order B by contributions; + D = limit C 3; + E = foreach D generate contributions; + STORE C INTO ':OUTPATH:.1'; + STORE E INTO ':OUTPATH:.2';?, + 'verify_pig_script' => q?A = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions); + B = foreach A generate age, registration,contributions; + C = order B by contributions; + STORE C INTO ':OUTPATH:.1'; + F = LOAD ':INPATH:/singlefile/votertab10k' AS (name, age, registration, contributions); + G = foreach F generate age, registration,contributions; + H = order G by contributions; + I = limit H 3; + J = foreach I generate contributions; + STORE J INTO ':OUTPATH:.2';?, } ], },{