pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1044496 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java test/org/apache/pig/test/TestEvalPipeline2.java
Date Fri, 10 Dec 2010 20:00:57 GMT
Author: daijy
Date: Fri Dec 10 20:00:57 2010
New Revision: 1044496

URL: http://svn.apache.org/viewvc?rev=1044496&view=rev
Log:
PIG-1761: New logical plan: Exception when bag dereference in the middle of expression

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
    pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1044496&r1=1044495&r2=1044496&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Dec 10 20:00:57 2010
@@ -59,6 +59,8 @@ Release 0.8.0 - Unreleased
 
 INCOMPATIBLE CHANGES
 
+PIG-1518: multi file input format for loaders (yanz via rding)
+
 PIG-1249: Safe-guards against misconfigured Pig scripts without PARALLEL keyword (zjffdu
vi olgan)
 
 IMPROVEMENTS
@@ -100,8 +102,6 @@ PIG-1483: [piggybank] Add HadoopJobHisto
 
 PIG-1555: [piggybank] add CSV Loader (dvryaboy)
 
-PIG-1518: multi file input format for loaders (yanz via rding)
-
 PIG-1501: need to investigate the impact of compression on pig performance (yanz via thejas)
 
 PIG-1497: Mandatory rule PartitionFilterOptimizer (xuefuz via daijy)
@@ -234,6 +234,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
+
 PIG-1757: After split combination, the number of maps may vary slightly (yanz)
 
 PIG-1760: Need to report progress in all databags (rding)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java?rev=1044496&r1=1044495&r2=1044496&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java Fri Dec 10 20:00:57
2010
@@ -136,11 +136,12 @@ public class ForeachInnerPlanVisitor ext
             } 
         }
 
-        // This case occurs when there are two projects one after another
+        // If project is after an expression, translate it into a dereference.
+        // One particular case is when there are two projects one after another
         // These projects in combination project a column (bag) out of a tuple 
         // and then project a column out of this projected bag
         // Here we merge these two projects into one BagDereferenceExpression
-        else if( op instanceof LOProject ) {
+        else if( op instanceof ExpressionOperator ) {
             LogicalExpression expOper = exprOpsMap.get(op);
             
             if (expOper!=null) {
@@ -152,6 +153,7 @@ public class ForeachInnerPlanVisitor ext
                 exprPlan.connect(dereferenceExp, expOper);
             }
         } else {
+            // If project is after a relational operator
             if (op instanceof RelationalOperator && project.isSendEmptyBagOnEOP())
{
                 LogicalOperator currentOp = op;
                 while (currentOp instanceof RelationalOperator) {

Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1044496&r1=1044495&r2=1044496&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Fri Dec 10 20:00:57 2010
@@ -966,4 +966,21 @@ public class TestEvalPipeline2 extends T
         
         fail();
     }
+    
+    // See PIG-1761
+    @Test
+    public void testBagDereferenceInMiddle() throws Exception{
+        String[] input1 = {
+                "foo@apache#44",
+        };
+        
+        Util.createInputFile(cluster, "table_testBagDereferenceInMiddle", input1);
+        pigServer.registerQuery("a = load 'table_testBagDereferenceInMiddle' as (a0:chararray);");
+        pigServer.registerQuery("b = foreach a generate UPPER(REGEX_EXTRACT_ALL(a0, '.*@(.*)#.*').$0);");
+        
+        Iterator<Tuple> iter = pigServer.openIterator("b");
+        Tuple t = iter.next();
+        assertTrue(t.size()==1);
+        assertTrue(t.get(0).equals("APACHE"));
+    }
 }



Mime
View raw message