pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1196466 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/newplan/logical/optimizer/ test/org/apache/pig/test/
Date Wed, 02 Nov 2011 06:18:53 GMT
Author: daijy
Date: Wed Nov  2 06:18:53 2011
New Revision: 1196466

URL: http://svn.apache.org/viewvc?rev=1196466&view=rev
Log:
PIG-2119: DuplicateForEachColumnRewrite makes assumptions about the position of LOGGenerate
in the plan

Added:
    pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
    pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
    pig/trunk/test/org/apache/pig/test/Util.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Nov  2 06:18:53 2011
@@ -156,6 +156,8 @@ PIG-2228: support partial aggregation in
 
 BUG FIXES
 
+PIG-2119: DuplicateForEachColumnRewrite makes assumptions about the position of LOGGenerate
in the plan (daijy)
+
 PIG-2290: TOBAG wraps tuple parameters in another tuple (ryan.hoegg via thejas)
 
 PIG-2288: Pig 0.9 error message not useful as compared to 0.8 in case 

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Wed
Nov  2 06:18:53 2011
@@ -48,6 +48,7 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.impl.util.ObjectSerializer;
 import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
 import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
 import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
 import org.apache.pig.newplan.logical.optimizer.UidResetter;
@@ -236,6 +237,9 @@ public class HExecutionEngine {
             pod.visit();
         }
         
+        DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover(
plan );
+        DanglingNestedNodeRemover.visit();
+        
         UidResetter uidResetter = new UidResetter( plan );
         uidResetter.visit();
         

Added: pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java?rev=1196466&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
(added)
+++ pig/trunk/src/org/apache/pig/newplan/logical/optimizer/DanglingNestedNodeRemover.java
Wed Nov  2 06:18:53 2011
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.newplan.logical.optimizer;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.DependencyOrderWalker;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOGenerate;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+
+public class DanglingNestedNodeRemover extends LogicalRelationalNodesVisitor {
+
+    public DanglingNestedNodeRemover(OperatorPlan plan)
+            throws FrontendException {
+        super(plan, new DependencyOrderWalker(plan));
+    }
+
+    @Override
+    public void visit(LOForEach foreach) throws FrontendException {
+        LogicalPlan innerPlan = foreach.getInnerPlan();
+        List<Operator> opsToRemove = new ArrayList<Operator>();
+        Iterator<Operator> ops = innerPlan.getOperators();
+        
+        while (ops.hasNext()) {
+            Operator op = ops.next();
+            // Check if op leads to LOGenerate, otherwise, candidate to remove
+            Operator currentOp = op;
+            boolean endWithNoLOGenerate = false;
+            while (!(currentOp instanceof LOGenerate)) {
+                if (innerPlan.getSuccessors(currentOp)==null) {
+                    endWithNoLOGenerate = true;
+                    break;
+                }
+                currentOp = innerPlan.getSuccessors(currentOp).get(0);
+            }
+            if (endWithNoLOGenerate)
+                opsToRemove.add(op);
+        }
+        
+        for (Operator op : opsToRemove) {
+            innerPlan.removeAndReconnect(op);
+        }
+    }
+}

Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Wed Nov  2 06:18:53 2011
@@ -175,4 +175,20 @@ public class TestPlanGeneration extends 
         poStore = (POStore)mrOper.mapPlan.getLeaves().get(0);
         assert(poStore.getAlias().equals("B"));
     }
+    
+    // See PIG-2119
+    @Test
+    public void testDanglingNestedNode() throws Exception  {
+        String query = "a = load 'b.txt' AS (id:chararray, num:int); " +
+            "b = group a by id;" +
+            "c = foreach b {" +
+            "  d = order a by num DESC;" +
+            "  n = COUNT(a);" +
+            "  e = limit d 1;" +
+            "  generate n;" +
+            "};";
+        
+        LogicalPlan lp = Util.parse(query, pc);
+        Util.optimizeNewLP(lp);
+    }
 }

Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1196466&r1=1196465&r2=1196466&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Wed Nov  2 06:18:53 2011
@@ -82,6 +82,7 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
 import org.apache.pig.impl.plan.CompilationMessageCollector;
 import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
 import org.apache.pig.newplan.logical.optimizer.LogicalPlanPrinter;
 import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
 import org.apache.pig.newplan.logical.relational.LOStore;
@@ -721,6 +722,9 @@ public class Util {
     public static  LogicalPlan optimizeNewLP( 
             LogicalPlan lp)
     throws FrontendException{
+        DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover(
lp );
+        DanglingNestedNodeRemover.visit();
+        
         UidResetter uidResetter = new UidResetter( lp );
         uidResetter.visit();
         



Mime
View raw message