pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1044495 - in /pig/branches/branch-0.8: CHANGES.txt src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java test/org/apache/pig/test/TestEvalPipeline2.java
Date Fri, 10 Dec 2010 19:59:31 GMT
Author: daijy
Date: Fri Dec 10 19:59:31 2010
New Revision: 1044495

URL: http://svn.apache.org/viewvc?rev=1044495&view=rev
Log:
PIG-1761: New logical plan: Exception when bag dereference in the middle of expression

Modified:
    pig/branches/branch-0.8/CHANGES.txt
    pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
    pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java

Modified: pig/branches/branch-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1044495&r1=1044494&r2=1044495&view=diff
==============================================================================
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Fri Dec 10 19:59:31 2010
@@ -22,6 +22,8 @@ Release 0.8.0
 
 INCOMPATIBLE CHANGES
 
+PIG-1518: multi file input format for loaders (yanz via rding)
+
 PIG-1249: Safe-guards against misconfigured Pig scripts without PARALLEL keyword (zjffdu
vi olgan)
 
 IMPROVEMENTS
@@ -77,8 +79,6 @@ PIG-1483: [piggybank] Add HadoopJobHisto
 
 PIG-1555: [piggybank] add CSV Loader (dvryaboy)
 
-PIG-1518: multi file input format for loaders (yanz via rding)
-
 PIG-1501: need to investigate the impact of compression on pig performance (yanz via thejas)
 
 PIG-1497: Mandatory rule PartitionFilterOptimizer (xuefuz via daijy)
@@ -211,6 +211,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
 
 BUG FIXES
 
+PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
+
 PIG-1760: Need to report progress in all databags (rding)
 
 PIG-1709: Skewed join use fewer reducer for extreme large key (daijy)

Modified: pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java?rev=1044495&r1=1044494&r2=1044495&view=diff
==============================================================================
--- pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
(original)
+++ pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
Fri Dec 10 19:59:31 2010
@@ -136,11 +136,12 @@ public class ForeachInnerPlanVisitor ext
             } 
         }
 
-        // This case occurs when there are two projects one after another
+        // If project is after an expression, translate it into a dereference.
+        // One particular case is when there are two projects one after another
         // These projects in combination project a column (bag) out of a tuple 
         // and then project a column out of this projected bag
         // Here we merge these two projects into one BagDereferenceExpression
-        else if( op instanceof LOProject ) {
+        else if( op instanceof ExpressionOperator ) {
             LogicalExpression expOper = exprOpsMap.get(op);
             
             if (expOper!=null) {
@@ -152,6 +153,7 @@ public class ForeachInnerPlanVisitor ext
                 exprPlan.connect(dereferenceExp, expOper);
             }
         } else {
+            // If project is after a relational operator
             if (op instanceof RelationalOperator && project.isSendEmptyBagOnEOP())
{
                 LogicalOperator currentOp = op;
                 while (currentOp instanceof RelationalOperator) {

Modified: pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1044495&r1=1044494&r2=1044495&view=diff
==============================================================================
--- pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java Fri Dec 10 19:59:31
2010
@@ -948,4 +948,21 @@ public class TestEvalPipeline2 extends T
         
         assertFalse(iter.hasNext());
     }
+    
+    // See PIG-1761
+    @Test
+    public void testBagDereferenceInMiddle() throws Exception{
+        String[] input1 = {
+                "foo@apache#44",
+        };
+        
+        Util.createInputFile(cluster, "table_testBagDereferenceInMiddle", input1);
+        pigServer.registerQuery("a = load 'table_testBagDereferenceInMiddle' as (a0:chararray);");
+        pigServer.registerQuery("b = foreach a generate UPPER(REGEX_EXTRACT_ALL(a0, '.*@(.*)#.*').$0);");
+        
+        Iterator<Tuple> iter = pigServer.openIterator("b");
+        Tuple t = iter.next();
+        assertTrue(t.size()==1);
+        assertTrue(t.get(0).equals("APACHE"));
+    }
 }



Mime
View raw message