pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1577022 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/relational/LOGenerate.java src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
Date Thu, 13 Mar 2014 02:54:26 GMT
Author: daijy
Date: Thu Mar 13 02:54:26 2014
New Revision: 1577022

URL: http://svn.apache.org/r1577022
Log:
PIG-3782: PushDownForEachFlatten + ColumnMapKeyPrune with user defined schema failing due
to incorrect UID assignment

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
    pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
    pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1577022&r1=1577021&r2=1577022&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 13 02:54:26 2014
@@ -245,6 +245,8 @@ PIG-3480: TFile-based tmpfile compressio
 
 BUG FIXES
 
+PIG-3782: PushDownForEachFlatten + ColumnMapKeyPrune with user defined schema failing due
to incorrect UID assignment (knoguchi via daijy)
+
 PIG-3779: Assert constructs ConstantExpression with null when no comment is given (thedatachef
via cheolsoo)
 
 PIG-3777: Pig 12.0 Documentation (karinahauser via daijy)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java?rev=1577022&r1=1577021&r2=1577022&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java Thu Mar 13 02:54:26
2014
@@ -33,6 +33,8 @@ import org.apache.pig.newplan.logical.re
 public class LOGenerate extends LogicalRelationalOperator {
      private List<LogicalExpressionPlan> outputPlans;
      private boolean[] flattenFlags;
+     // mUserDefinedSchema is the original input from the user, we don't suppose
+     // to store uid in mUserDefinedSchema
      private List<LogicalSchema> mUserDefinedSchema = null;
      private List<LogicalSchema> outputPlanSchemas = null;
      // If LOGenerate generate new uid, cache it here.

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java?rev=1577022&r1=1577021&r2=1577022&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java Thu Mar
13 02:54:26 2014
@@ -260,6 +260,7 @@ public class PushDownForEachFlatten exte
                                 fieldsToBeFlattaned.add(fieldCount);
                                 if (gen.getUserDefinedSchema()!=null && gen.getUserDefinedSchema().get(i)!=null)
{
                                     cachedUserDefinedSchema.put(fieldCount, gen.getUserDefinedSchema().get(i));
+                                    cachedUserDefinedSchema.get(fieldCount).mergeUid(gen.getOutputPlanSchemas().get(i));
                                     gen.getUserDefinedSchema().set(i, null);
                                 }
                                 fieldCount++;

Modified: pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java?rev=1577022&r1=1577021&r2=1577022&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java Thu Mar 13 02:54:26
2014
@@ -1111,6 +1111,25 @@ public class TestNewPlanPushDownForeachF
                   ((LOLoad)load).getSchema().getField("a1") != null );
     }
 
+    // See PIG-3782
+    @Test
+    public void testForeachJoinWithUserDefinedSchemaAndPruning() throws Exception {
+        String query =
+        "a = load '1.txt' as (a0:int, a1, a2:bag{});" +
+        "b = load '2.txt' as (b0:int, b1);" +
+        "c = foreach a generate a0, flatten(a2) as (q1, q2);" +
+        "d = join c by a0, b by b0;" +
+        "e = foreach d generate a0, q1, q2;" +
+        "f = foreach e generate a0, (int)q1, (int)q2;" +
+        "store f into 'output';" ;
+
+        LogicalPlan newLogicalPlan = migrateAndOptimizePlanWithPruning( query );
+        //In the original issue, Exception is thrown during the Pruning due to
+        //incorrect UID assignment by the PushDownForEachFlatten failing this
+        //test
+
+    }
+
     public class MyPlanOptimizerWithPruning extends LogicalPlanOptimizer {
         protected MyPlanOptimizerWithPruning (OperatorPlan p,  int iterations) {
             super(p, iterations, new HashSet<String>());



Mime
View raw message