pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1198061 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java test/org/apache/pig/test/TestPlanGeneration.java
Date Sat, 05 Nov 2011 21:16:13 GMT
Author: daijy
Date: Sat Nov  5 21:16:12 2011
New Revision: 1198061

URL: http://svn.apache.org/viewvc?rev=1198061&view=rev
Log:
PIG-2339: HCatLoader loads all the partitions in a partitioned table even though a filter
clause on the partitions is specified in the Pig script

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
    pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1198061&r1=1198060&r2=1198061&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Nov  5 21:16:12 2011
@@ -292,6 +292,9 @@ PIG-2320: Error: "projection with nothin
 
 PIG-2346: TypeCastInsert should not insert Foreach if there is no as statement (daijy)
 
+PIG-2339: HCatLoader loads all the partitions in a partitioned table even though
+ a filter clause on the partitions is specified in the Pig script (daijy)
+
 Release 0.9.1 - Unreleased
 
 IMPROVEMENTS

Modified: pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java?rev=1198061&r1=1198060&r2=1198061&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java Sat Nov
 5 21:16:12 2011
@@ -91,15 +91,6 @@ public class LogicalPlanOptimizer extend
         checkAndAddRule(s, r);
         if (!s.isEmpty())
             ls.add(s);
-        
-        // Partition filter set
-        // This set of rules push partition filter to LoadFunc
-        s = new HashSet<Rule>();
-        // Optimize partition filter
-        r = new PartitionFilterOptimizer("PartitionFilterOptimizer");
-        checkAndAddRule(s, r);
-        if (!s.isEmpty())
-            ls.add(s);
 
         // Limit Set
         // This set of rules push up limit
@@ -141,6 +132,15 @@ public class LogicalPlanOptimizer extend
         if (!s.isEmpty())
             ls.add(s);
         
+        // Partition filter set
+        // This set of rules push partition filter to LoadFunc
+        s = new HashSet<Rule>();
+        // Optimize partition filter
+        r = new PartitionFilterOptimizer("PartitionFilterOptimizer");
+        checkAndAddRule(s, r);
+        if (!s.isEmpty())
+            ls.add(s);
+        
         // PushDownForEachFlatten set
         s = new HashSet<Rule>();
         // Add the PushDownForEachFlatten

Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1198061&r1=1198060&r2=1198061&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Sat Nov  5 21:16:12 2011
@@ -251,4 +251,64 @@ public class TestPlanGeneration extends 
         
         Assert.assertTrue(lp.getSuccessors(loStore)==null);
     }
+    
+    public static class PartitionedLoader extends PigStorage implements LoadMetadata {
+
+        Schema schema;
+        String[] partCols;
+        static Expression partFilter = null;
+        
+        public PartitionedLoader(String schemaString, String commaSepPartitionCols) 
+        throws ParserException {
+            schema = Utils.getSchemaFromString(schemaString);
+            partCols = commaSepPartitionCols.split(",");
+        }
+
+        @Override
+        public ResourceSchema getSchema(String location, Job job)
+        throws IOException {
+            return new ResourceSchema(schema);
+        }
+
+        @Override
+        public ResourceStatistics getStatistics(String location,
+                Job job) throws IOException {
+            return null;
+        }
+
+        @Override
+        public void setPartitionFilter(Expression partitionFilter)
+        throws IOException {
+            partFilter = partitionFilter;            
+        }
+
+        @Override
+        public String[] getPartitionKeys(String location, Job job)
+                throws IOException {
+            return partCols;
+        }
+        
+        public Expression getPartFilter() {
+            return partFilter;
+        }
+
+    }
+    
+    @Test
+    // See PIG-2339
+    public void testPartitionFilterOptimizer() throws Exception {
+        String query = "a = load 'foo' using " + PartitionedLoader.class.getName() + 
+                "('name:chararray, dt:chararray', 'dt');\n" +
+            "b = filter a by dt=='2011';\n" +
+            "store b into 'output';";
+            
+        LogicalPlan lp = Util.parse(query, pc);
+        Util.optimizeNewLP(lp);
+        
+        LOLoad loLoad = (LOLoad)lp.getSources().get(0);
+        LOForEach loForEach = (LOForEach)lp.getSuccessors(loLoad).get(0);
+        LOStore loStore = (LOStore)lp.getSuccessors(loForEach).get(0);
+        Assert.assertTrue(((PartitionedLoader)loLoad.getLoadFunc()).getPartFilter()!=null);
+        Assert.assertTrue(loStore.getAlias().equals("b"));
+    }
 }



Mime
View raw message