pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1176200 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/parser/ test/org/apache/pig/parser/ test/org/apache/pig/test/
Date Tue, 27 Sep 2011 05:28:53 GMT
Author: thejas
Date: Tue Sep 27 05:28:52 2011
New Revision: 1176200

URL: http://svn.apache.org/viewvc?rev=1176200&view=rev
Log:
PIG-2223: error accessing column in output schema of udf having project-star input (thejas)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/PigServer.java
    pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
    pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java
    pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java
    pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
    pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Sep 27 05:28:52 2011
@@ -134,6 +134,8 @@ PIG-2228: support partial aggregation in
 
 BUG FIXES
 
+PIG-2223: error accessing column in output schema of udf having project-star input (thejas)
+
 PIG-2273: Pig.compileFromFile in embedded python fails when pig script starts with a comment
(ddaniels888 via gates)
 
 PIG-2278: Wrong version numbers for libraries in eclipse template classpath (azaroth)

Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Tue Sep 27 05:28:52 2011
@@ -83,6 +83,7 @@ import org.apache.pig.newplan.logical.ex
 import org.apache.pig.newplan.logical.expression.ScalarExpression;
 import org.apache.pig.newplan.logical.optimizer.AllExpressionVisitor;
 import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
 import org.apache.pig.newplan.logical.relational.LOForEach;
 import org.apache.pig.newplan.logical.relational.LOLoad;
 import org.apache.pig.newplan.logical.relational.LOStore;
@@ -1613,7 +1614,6 @@ public class PigServer {
         }
         
         private void compile(LogicalPlan lp) throws FrontendException  {
-            new ProjStarInUdfExpander(lp).visit();
             new ColumnAliasConversionVisitor( lp ).visit();
             new SchemaAliasVisitor( lp ).visit();
             new ScalarVisitor( lp, pigContext ).visit();

Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanBuilder.java Tue Sep 27 05:28:52 2011
@@ -63,6 +63,7 @@ import org.apache.pig.newplan.logical.ex
 import org.apache.pig.newplan.logical.expression.OrExpression;
 import org.apache.pig.newplan.logical.expression.ProjectExpression;
 import org.apache.pig.newplan.logical.expression.UserFuncExpression;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
 import org.apache.pig.newplan.logical.relational.LOCogroup;
 import org.apache.pig.newplan.logical.relational.LOCogroup.GROUPTYPE;
 import org.apache.pig.newplan.logical.relational.LOCross;
@@ -87,6 +88,7 @@ import org.apache.pig.newplan.logical.re
 import org.apache.pig.newplan.logical.relational.LogicalSchema;
 import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
 import org.apache.pig.newplan.logical.rules.OptimizerUtils;
+import org.apache.pig.newplan.logical.visitor.ProjStarInUdfExpander;
 import org.apache.pig.newplan.logical.visitor.ProjectStarExpander;
 
 public class LogicalPlanBuilder {
@@ -158,9 +160,19 @@ public class LogicalPlanBuilder {
         return new LOFilter( plan, true );
     }
     
-    String buildFilterOp(SourceLocation loc, LOFilter op, String alias, String inputAlias,
LogicalExpressionPlan expr) throws ParserValidationException {
+    String buildFilterOp(SourceLocation loc, LOFilter op, String alias, 
+            String inputAlias, LogicalExpressionPlan expr)
+                    throws ParserValidationException {
+        
         op.setFilterPlan( expr );
-        return buildOp( loc, op, alias, inputAlias, null );
+        alias = buildOp( loc, op, alias, inputAlias, null ); // it should actually return
same alias 
+        try {
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
+        } catch (FrontendException e) {
+            throw new ParserValidationException( intStream, loc, e );
+        }   
+        return alias;
     }
     
     String buildDistinctOp(SourceLocation loc, String alias, String inputAlias, String partitioner)
throws ParserValidationException {
@@ -179,7 +191,9 @@ public class LogicalPlanBuilder {
     }
     
     String buildSampleOp(SourceLocation loc, String alias, String inputAlias, double value,
-            SourceLocation valLoc) throws ParserValidationException {
+            SourceLocation valLoc)
+                    throws ParserValidationException {
+        
         LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
         //  Generate a filter condition.
         LogicalExpression konst = new ConstantExpression( filterPlan, value);
@@ -191,7 +205,9 @@ public class LogicalPlanBuilder {
     }
     
     String buildSampleOp(SourceLocation loc, LOFilter filter, String alias, String inputAlias,
-            LogicalExpressionPlan samplePlan, LogicalExpression expr) throws ParserValidationException
{
+            LogicalExpressionPlan samplePlan, LogicalExpression expr)
+                    throws ParserValidationException {
+        
         UserFuncExpression udf = new UserFuncExpression( samplePlan, new FuncSpec( RANDOM.class.getName()
) );
         new LessThanExpression( samplePlan, udf, expr );
         return buildFilterOp( loc, filter, alias, inputAlias, samplePlan );
@@ -280,6 +296,8 @@ public class LogicalPlanBuilder {
         alias = buildOp( loc, sort, alias, inputAlias, null );
         try {
             (new ProjectStarExpander(sort.getPlan())).visit(sort);
+            (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);
+            new SchemaResetter(sort.getPlan(), true).visit(sort);
         } catch (FrontendException e) {
             throw new ParserValidationException( intStream, loc, e );
         }
@@ -339,6 +357,8 @@ public class LogicalPlanBuilder {
         alias = buildOp( loc, op, alias, inputAliases, partitioner );
         try {
             (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
         } catch (FrontendException e) {
             throw new ParserValidationException( intStream, loc, e );
         }
@@ -380,6 +400,8 @@ public class LogicalPlanBuilder {
         alias = buildOp( loc, op, alias, inputAliases, partitioner );
         try {
             (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
         } catch (FrontendException e) {
             throw new ParserValidationException( intStream, loc, e );
         }
@@ -494,6 +516,8 @@ public class LogicalPlanBuilder {
         alias = buildOp( loc, op, alias, inputAlias, null );
         try {
             (new ProjectStarExpander(op.getPlan())).visit(op);
+            (new ProjStarInUdfExpander(op.getPlan())).visit(op);
+            new SchemaResetter(op.getPlan(), true).visit(op);
         } catch (FrontendException e) {
             throw new ParserValidationException( intStream, loc, e );
         }

Modified: pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestColumnAliasConversion.java Tue Sep 27 05:28:52
2011
@@ -101,7 +101,7 @@ public class TestColumnAliasConversion {
                        "C = store B into 'output';";
         try {
             validate( query );
-        } catch(PlanValidationException ex) {
+        }catch(ParserValidationException ex){
             return;
         }
         Assert.fail( "Query should fail to validate." );
@@ -140,7 +140,7 @@ public class TestColumnAliasConversion {
                        "C = store B into 'output';";
         try {
             validate( query );
-        } catch(PlanValidationException ex) {
+        } catch(ParserValidationException ex) {
             return;
         }
         Assert.fail( "Query should fail to validate." );

Modified: pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestDataBagAccess.java Tue Sep 27 05:28:52 2011
@@ -130,7 +130,7 @@ public class TestDataBagAccess extends T
         } catch(FrontendException e) {
             exceptionOccured = true;
             String msg = e.getMessage();
-            Util.checkStrContainsSubStr(msg, "Invalid field reference. Referenced field [t]
does not exist in schema");
+            Util.checkStrContainsSubStr(msg, "Cannot find field t in i:int,d:double,c:chararray");
         }
         assertTrue(exceptionOccured);
     }

Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Tue Sep 27 05:28:52 2011
@@ -898,8 +898,8 @@ public class TestEvalPipeline2 {
             pigServer.openIterator("c");
         } catch (Exception e) {
             PigException pe = LogUtils.getPigException(e);
-            Assert.assertTrue(pe.getErrorCode()==1031);
-            Assert.assertTrue(pe.getMessage().contains("Incompatable schema"));
+            Util.checkStrContainsSubStr(pe.getMessage(), "ERROR 1031");
+            Util.checkStrContainsSubStr(pe.getMessage(), "Incompatable schema");
             return;
         }
         Assert.fail();

Modified: pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1176200&r1=1176199&r2=1176200&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Tue Sep 27 05:28:52
2011
@@ -145,6 +145,33 @@ public class TestProjectStarRangeInUdf  
 
     }
 
+    //PIG-2223 
+    // lookup on column name in udf output tuple schema
+    @Test
+    public void testProjStarExpandInForeachLookup1() throws IOException {
+
+        String query =
+            "  l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int);"
+            + "f = foreach l1 generate TOTUPLE(*) as tb;"
+            + "f2 = foreach f generate tb.a, tb.b;"                    
+            ; 
+        compileAndCompareSchema("a : int, b : int", query, "f2");
+    }
+    
+    //PIG-2223 
+    // lookup on column name in udf output tuple schema
+    @Test
+    public void testProjStarExpandInForeachLookup2() throws IOException {
+
+        String query =
+            "  l1 = load '" + INP_FILE_5FIELDS + "' as (a : int, b : int, c : int);"
+            + "f = foreach l1 generate TOTUPLE(b .. ) as tb;"
+            + "f2 = foreach f generate tb.b as b2, tb.c as c2;"     
+            + "f3 = foreach f2 generate b2, b2 + c2 as bc2;"     
+            ; 
+        compileAndCompareSchema("b2 : int, bc2 : int", query, "f3");
+    }
+    
     @Test
     public void testProjStarExpandInFilter1() throws IOException{
         //TOBAG has * and a bincond expression as argument



Mime
View raw message