pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1148106 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/relational/LOUnion.java test/org/apache/pig/parser/TestUnionOnSchemaSetter.java test/org/apache/pig/test/TestEvalPipeline2.java
Date Mon, 18 Jul 2011 23:47:55 GMT
Author: daijy
Date: Mon Jul 18 23:47:54 2011
New Revision: 1148106

URL: http://svn.apache.org/viewvc?rev=1148106&view=rev
Log:
PIG-2159: New logical plan uses incorrect class for SUM causing for ClassCastException

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/relational/LOUnion.java
    pig/trunk/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java
    pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1148106&r1=1148105&r2=1148106&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Jul 18 23:47:54 2011
@@ -314,6 +314,8 @@ PIG-1696: Performance: Use System.arrayc
 
 BUG FIXES
 
+PIG-2159: New logical plan uses incorrect class for SUM causing for ClassCastException (daijy)
+
 PIG-2106: Fix Zebra unit test TestBasicUnion.testNeg3, TestBasicUnion.testNeg4 (daijy)
 
 PIG-2083: bincond ERROR 1025: Invalid field projection when null is used (thejas)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOUnion.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOUnion.java?rev=1148106&r1=1148105&r2=1148106&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOUnion.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOUnion.java Mon Jul 18 23:47:54
2011
@@ -97,30 +97,48 @@ public class LOUnion extends LogicalRela
                 if (mergedSchema == null)
                     return null;
             }
-            
-            // Bring back cached uid if any; otherwise, cache uid generated
-            for (int i=0;i<s0.size();i++)
-            {
-                LogicalSchema.LogicalFieldSchema fs = mergedSchema.getField(i);
-                long uid = -1;
-                for (Pair<Long, Long> pair : uidMapping) {
-                    if (pair.second==s0.getField(i).uid) {
-                        uid = pair.first;
-                        break;
-                    }
+        }
+
+        // Bring back cached uid if any; otherwise, cache uid generated
+        for (int i=0;i<s0.size();i++)
+        {
+            LogicalSchema.LogicalFieldSchema outputFieldSchema;
+            if (onSchema) {
+                outputFieldSchema = mergedSchema.getFieldSubNameMatch(s0.getField(i).alias);
+            } else {
+                outputFieldSchema = mergedSchema.getField(i);
+            }
+            long uid = -1;
+            for (Pair<Long, Long> pair : uidMapping) {
+                if (pair.second==s0.getField(i).uid) {
+                    uid = pair.first;
+                    break;
                 }
-                if (uid==-1) {
-                    uid = LogicalExpression.getNextUid();
-                    for (Operator input : inputs) {
-                        long inputUid = ((LogicalRelationalOperator)input).getSchema().getField(i).uid;
-                        uidMapping.add(new Pair<Long, Long>(uid, inputUid));
+            }
+            if (uid==-1) {
+                uid = LogicalExpression.getNextUid();
+                for (Operator input : inputs) {
+                    long inputUid;
+                    LogicalFieldSchema matchedInputFieldSchema;
+                	if (onSchema) {
+                	    matchedInputFieldSchema = ((LogicalRelationalOperator)input).getSchema().getFieldSubNameMatch(s0.getField(i).alias);
+                        if (matchedInputFieldSchema!=null) {
+                            inputUid = matchedInputFieldSchema.uid;
+                            uidMapping.add(new Pair<Long, Long>(uid, inputUid));
+                        }
                     }
+                    else {
+                        matchedInputFieldSchema = mergedSchema.getField(i);
+	                	inputUid = ((LogicalRelationalOperator)input).getSchema().getField(i).uid;
+	                	uidMapping.add(new Pair<Long, Long>(uid, inputUid));
+                    }
+                	
                 }
-
-                fs.uid = uid;
             }
-        }
 
+            outputFieldSchema.uid = uid;
+        }
+        
         return schema = mergedSchema;
     }
 

Modified: pig/trunk/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java?rev=1148106&r1=1148105&r2=1148106&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java Mon Jul 18 23:47:54
2011
@@ -24,7 +24,10 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
 
+@RunWith(JUnit4.class)
 public class TestUnionOnSchemaSetter {
     @Test
     public void test1() throws FrontendException {

Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1148106&r1=1148105&r2=1148106&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Mon Jul 18 23:47:54 2011
@@ -1542,4 +1542,47 @@ public class TestEvalPipeline2 {
         
         pigServer.getPigContext().getProperties().remove("pig.optimizer.rules");
     }
+    
+    // See PIG-2159
+    @Test
+    public void testUnionOnSchemaUidGeneration() throws Exception{
+        String[] input1 = {
+        		"100,101,102,103,104,105",
+        		"110,111,112,113,114,115"
+        };
+        
+        String[] input2 = {
+        		"200,201,202,203,204,205",
+        		"210,211,212,213,214,215"
+        };
+        
+        String[] input0 = {
+        		"200,201,202,203,204,205",
+        		"210,211,212,213,214,215"
+        };
+        
+        Util.createInputFile(cluster, "table_testUnionOnSchemaUidGeneration1", input1);
+        Util.createInputFile(cluster, "table_testUnionOnSchemaUidGeneration2", input2);
+        Util.createInputFile(cluster, "table_testUnionOnSchemaUidGeneration0", input0);
+        
+        pigServer.registerQuery("A = load 'table_testUnionOnSchemaUidGeneration1' using PigStorage(',')
 as (f1:int,f2:int,f3:int,f4:long,f5:double);");
+        pigServer.registerQuery("B = load 'table_testUnionOnSchemaUidGeneration2' using PigStorage(',')
 as (f1:int,f2:int,f3:int,f4:long,f5:double);");
+        pigServer.registerQuery("C = load 'table_testUnionOnSchemaUidGeneration0' using PigStorage(',')
 as (f1:int,f2:int,f3:int);");
+        pigServer.registerQuery("U = UNION ONSCHEMA A,B;");
+        pigServer.registerQuery("J = join C by (f1,f2,f3) LEFT OUTER, U by (f1,f2,f3);");
+        pigServer.registerQuery("Porj = foreach J generate C::f1 as f1 ,C::f2 as f2,C::f3
as f3,U::f4 as f4,U::f5 as f5;");
+        pigServer.registerQuery("G = GROUP Porj by (f1,f2,f3,f5);");
+        pigServer.registerQuery("Final = foreach G generate SUM(Porj.f4) as total;");
+
+        Iterator<Tuple> iter = pigServer.openIterator("Final");
+        
+        Tuple t = iter.next();
+        Assert.assertTrue(t.toString().equals("(203)"));
+        
+        t = iter.next();
+        Assert.assertTrue(t.toString().equals("(213)"));
+        
+        Assert.assertFalse(iter.hasNext());
+        
+    }
 }



Mime
View raw message