pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1198031 - in /pig/trunk: ./ src/org/apache/pig/newplan/logical/relational/ src/org/apache/pig/newplan/logical/rules/ test/org/apache/pig/test/
Date Sat, 05 Nov 2011 19:34:33 GMT
Author: daijy
Date: Sat Nov  5 19:34:32 2011
New Revision: 1198031

URL: http://svn.apache.org/viewvc?rev=1198031&view=rev
Log:
PIG-2346: TypeCastInsert should not insert Foreach if there is no as statement

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java
    pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java
    pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java
    pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
    pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Nov  5 19:34:32 2011
@@ -290,6 +290,8 @@ BUG FIXES
 
 PIG-2320: Error: "projection with nothing to reference" (daijy)
 
+PIG-2346: TypeCastInsert should not insert Foreach if there is no as statement (daijy)
+
 Release 0.9.1 - Unreleased
 
 IMPROVEMENTS

Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java Sat Nov  5 19:34:32
2011
@@ -278,4 +278,8 @@ public class LOLoad extends LogicalRelat
         this.signature = signature;
         loadFunc.setUDFContextSignature(signature);
     }
+    
+    public LogicalSchema getScriptSchema() {
+        return scriptSchema;
+    }
 }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java Sat Nov  5 19:34:32
2011
@@ -113,4 +113,8 @@ public class LOStream extends LogicalRel
     public void resetUid() {
         uidOnlySchema = null;
     }
+    
+    public LogicalSchema getScriptSchema() {
+        return scriptSchema;
+    }
 }

Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java Sat Nov  5 19:34:32
2011
@@ -77,9 +77,11 @@ public abstract class TypeCastInserter e
             if (s == null) return false;
     
             if (op instanceof LOLoad) {
+                if (((LOLoad)op).getScriptSchema()==null) return false;
                 if (((LOLoad)op).isCastInserted()) return false;
             }
             else {
+                if (((LOStream)op).getScriptSchema()==null) return false;
                 if (((LOStream)op).isCastInserted()) return false;
             }
             

Modified: pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java Sat Nov  5 19:34:32
2011
@@ -568,7 +568,7 @@ public class TestPartitionFilterPushDown
             Assert.assertTrue(!(iter.next() instanceof LOFilter));
             counter++;
         }      
-        Assert.assertEquals(counter, 7);
+        Assert.assertEquals(counter, 5);
     }
 
     //// helper methods ///////

Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Sat Nov  5 19:34:32 2011
@@ -17,11 +17,18 @@
  */
 package org.apache.pig.test;
 
+import java.io.IOException;
 import java.util.Properties;
 
 import junit.framework.Assert;
 
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.pig.ExecType;
+import org.apache.pig.Expression;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.LoadMetadata;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.ResourceStatistics;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
@@ -29,17 +36,25 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.Utils;
 import org.apache.pig.newplan.Operator;
 import org.apache.pig.newplan.logical.expression.LogicalExpression;
 import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOFilter;
 import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOLoad;
 import org.apache.pig.newplan.logical.relational.LOSort;
 import org.apache.pig.newplan.logical.relational.LOStore;
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.newplan.logical.relational.LogicalSchema;
 import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+import org.apache.pig.parser.ParserException;
+import org.apache.pig.test.TestPartitionFilterPushDown.TestLoader;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -191,4 +206,49 @@ public class TestPlanGeneration extends 
         LogicalPlan lp = Util.parse(query, pc);
         Util.optimizeNewLP(lp);
     }
+    
+    public static class SchemaLoader extends PigStorage implements LoadMetadata {
+
+        Schema schema;
+        public SchemaLoader(String schemaString) throws ParserException {
+            schema = Utils.getSchemaFromString(schemaString);
+        }
+        @Override
+        public ResourceSchema getSchema(String location, Job job)
+                throws IOException {
+            return new ResourceSchema(schema);
+        }
+
+        @Override
+        public ResourceStatistics getStatistics(String location, Job job)
+                throws IOException {
+            return null;
+        }
+
+        @Override
+        public String[] getPartitionKeys(String location, Job job)
+                throws IOException {
+            return null;
+        }
+
+        @Override
+        public void setPartitionFilter(Expression partitionFilter)
+                throws IOException {
+        }
+    }
+    
+    @Test
+    public void testLoaderWithSchema() throws Exception {
+        String query = "a = load 'foo' using " + SchemaLoader.class.getName() + "('name,age,gpa');\n"
+                + "b = filter a by age==20;"
+                + "store b into 'output';";
+        LogicalPlan lp = Util.parse(query, pc);
+        Util.optimizeNewLP(lp);
+        
+        LOLoad loLoad = (LOLoad)lp.getSources().get(0);
+        LOFilter loFilter = (LOFilter)lp.getSuccessors(loLoad).get(0);
+        LOStore loStore = (LOStore)lp.getSuccessors(loFilter).get(0);
+        
+        Assert.assertTrue(lp.getSuccessors(loStore)==null);
+    }
 }



Mime
View raw message