pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1196748 [2/2] - in /pig/branches/branch-0.10: ./ ivy/ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/ shims/src/hadoop23/org/apache/pi...
Date Wed, 02 Nov 2011 18:33:15 GMT
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestSecondarySort.java?rev=1196748&r1=1196747&r2=1196748&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestSecondarySort.java Wed Nov  2 18:33:14
2011
@@ -38,6 +38,7 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
@@ -462,16 +463,20 @@ public class TestSecondarySort {
         ps1.println("1\t2\t4");
         ps1.println("2\t3\t4");
         ps1.close();
+        
+        String expected[] = {
+                "(2,{(2,3,4)})",
+                "(1,{(1,2,3),(1,2,4),(1,2,4),(1,2,4),(1,3,4)})"
+        };
+        
         Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
         pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1,
a2);");
         pigServer.registerQuery("B = group A by $0 parallel 2;");
         pigServer.registerQuery("C = foreach B { D = limit A 10; E = order D by $1; generate
group, E;};");
         Iterator<Tuple> iter = pigServer.openIterator("C");
-        assertTrue(iter.hasNext());
-        assertTrue(iter.next().toString().equals("(2,{(2,3,4)})"));
-        assertTrue(iter.hasNext());
-        assertTrue(iter.next().toString().equals("(1,{(1,2,3),(1,2,4),(1,2,4),(1,2,4),(1,3,4)})"));
-        assertFalse(iter.hasNext());
+        Schema s = pigServer.dumpSchema("C");
+        
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
     }
 
@@ -486,16 +491,20 @@ public class TestSecondarySort {
         ps1.println("1\t8\t4");
         ps1.println("2\t3\t4");
         ps1.close();
+        
+        String expected[] = {
+                "(2,{(2,3,4)})",
+                "(1,{(1,8,4),(1,4,4),(1,3,4),(1,2,3),(1,2,4)})"
+        };
+        
         Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
         pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1,
a2);");
         pigServer.registerQuery("B = group A by $0 parallel 2;");
         pigServer.registerQuery("C = foreach B { D = order A by a1 desc; generate group,
D;};");
         Iterator<Tuple> iter = pigServer.openIterator("C");
-        assertTrue(iter.hasNext());
-        assertEquals("(2,{(2,3,4)})", iter.next().toString());
-        assertTrue(iter.hasNext());
-        assertEquals("(1,{(1,8,4),(1,4,4),(1,3,4),(1,2,3),(1,2,4)})", iter.next().toString());
-        assertFalse(iter.hasNext());
+        Schema s = pigServer.dumpSchema("C");
+        
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
     }
 
@@ -571,18 +580,21 @@ public class TestSecondarySort {
         ps1.println("1\t2\t4");
         ps1.println("2\t3\t4");
         ps1.close();
+        
+        String expected[] = {
+                "((1,2),{(2,3),(2,4),(2,4),(2,4)})",
+                "((1,3),{(3,4)})",
+                "((2,3),{(3,4)})"
+        };
         Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
         pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1,
a2);");
         pigServer.registerQuery("B = group A by (a0, a1);");
         pigServer.registerQuery("C = foreach B { C1 = A.(a1,a2); generate group, C1;};");
         Iterator<Tuple> iter = pigServer.openIterator("C");
-        assertTrue(iter.hasNext());
-        assertEquals("((1,2),{(2,3),(2,4),(2,4),(2,4)})", iter.next().toString());
-        assertTrue(iter.hasNext());
-        assertEquals("((1,3),{(3,4)})", iter.next().toString());
-        assertTrue(iter.hasNext());
-        assertEquals("((2,3),{(3,4)})", iter.next().toString());
-        assertFalse(iter.hasNext());
+        Schema s = pigServer.dumpSchema("C");
+        
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
+        
         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
     }
 }
\ No newline at end of file

Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestStreaming.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestStreaming.java?rev=1196748&r1=1196747&r2=1196748&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestStreaming.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestStreaming.java Wed Nov  2 18:33:14
2011
@@ -214,7 +214,7 @@ public class TestStreaming {
             }
     		
     		// Run the query and check the results
-    		Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults);
+    		Util.checkQueryOutputsAfterSort(pigServer.openIterator("OP"), expectedResults);
         }
 	}
 

Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestStreamingLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestStreamingLocal.java?rev=1196748&r1=1196747&r2=1196748&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestStreamingLocal.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestStreamingLocal.java Wed Nov  2 18:33:14
2011
@@ -197,7 +197,7 @@ public class TestStreamingLocal extends 
             }
 
             // Run the query and check the results
-            Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults);
+            Util.checkQueryOutputsAfterSort(pigServer.openIterator("OP"), expectedResults);
         }
     }
 

Modified: pig/branches/branch-0.10/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/Util.java?rev=1196748&r1=1196747&r2=1196748&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/Util.java Wed Nov  2 18:33:14 2011
@@ -58,8 +58,10 @@ import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.log4j.SimpleLayout;
 import org.apache.pig.ExecType;
+import org.apache.pig.LoadCaster;
 import org.apache.pig.PigException;
 import org.apache.pig.PigServer;
+import org.apache.pig.ResourceSchema.ResourceFieldSchema;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
@@ -69,10 +71,13 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
 import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
+import org.apache.pig.builtin.Utf8StorageConverter;
 import org.apache.pig.data.BagFactory;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
+import org.apache.pig.data.DefaultBagFactory;
+import org.apache.pig.data.SortedDataBag;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.PigContext;
@@ -88,6 +93,8 @@ import org.apache.pig.newplan.logical.op
 import org.apache.pig.newplan.logical.relational.LOStore;
 import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
 import org.apache.pig.newplan.logical.optimizer.UidResetter;
 import org.apache.pig.newplan.logical.rules.LoadStoreFuncDupSignatureValidator;
 import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
@@ -1042,7 +1049,86 @@ public class Util {
         }
     }
 
-  
+    
+    static public void checkQueryOutputsAfterSort(Iterator<Tuple> actualResultsIt,

+            Tuple[] expectedResArray) {
+        List<Tuple> list = new ArrayList<Tuple>();
+        Collections.addAll(list, expectedResArray);
+        checkQueryOutputsAfterSort(actualResultsIt, list);
+    }
+     
+    
+    static private void convertBagToSortedBag(Tuple t) {
+        for (int i=0;i<t.size();i++) {
+           Object obj = null;
+           try {
+               obj = t.get(i);
+           } catch (ExecException e) {
+               // shall not happen
+           }
+           if (obj instanceof DataBag) {
+                DataBag bag = (DataBag)obj;
+                Iterator<Tuple> iter = bag.iterator();
+                DataBag sortedBag = DefaultBagFactory.getInstance().newSortedBag(null);
+                while (iter.hasNext()) {
+                    Tuple t2 = iter.next();
+                    sortedBag.add(t2);
+                    convertBagToSortedBag(t2);
+                }
+                try {
+                    t.set(i, sortedBag);
+                } catch (ExecException e) {
+                    // shall not happen
+                }
+           }
+        }
+    }
+    
+    static public void checkQueryOutputsAfterSortRecursive(Iterator<Tuple> actualResultsIt,

+            String[] expectedResArray, String schemaString) throws IOException {
+        LogicalSchema resultSchema = org.apache.pig.impl.util.Utils.parseSchema(schemaString);
+        checkQueryOutputsAfterSortRecursive(actualResultsIt, expectedResArray, resultSchema);
+    }
+          /**
+     * Helper function to check if the result of a Pig Query is in line with 
+     * expected results. It sorts actual and expected string results before comparison
+     * 
+     * @param actualResultsIt Result of the executed Pig query
+     * @param expectedResArray Expected string results to validate against
+     * @param fs fieldSchema of expecteResArray
+     * @throws IOException 
+     */
+    static public void checkQueryOutputsAfterSortRecursive(Iterator<Tuple> actualResultsIt,

+            String[] expectedResArray, LogicalSchema schema) throws IOException {
+        LogicalFieldSchema fs = new LogicalFieldSchema("tuple", schema, DataType.TUPLE);
+        ResourceFieldSchema rfs = new ResourceFieldSchema(fs);
+        
+        LoadCaster caster = new Utf8StorageConverter();
+        List<Tuple> actualResList = new ArrayList<Tuple>();
+        while(actualResultsIt.hasNext()){
+            actualResList.add(actualResultsIt.next());
+        }
+        
+        List<Tuple> expectedResList = new ArrayList<Tuple>();
+        for (String str : expectedResArray) {
+            Tuple newTuple = caster.bytesToTuple(str.getBytes(), rfs);
+            expectedResList.add(newTuple);
+        }
+        
+        for (Tuple t : actualResList) {
+            convertBagToSortedBag(t);
+        }
+        
+        for (Tuple t : expectedResList) {
+            convertBagToSortedBag(t);
+        }
+        
+        Collections.sort(actualResList);
+        Collections.sort(expectedResList);
+        
+        Assert.assertEquals("Comparing actual and expected results. ",
+                expectedResList, actualResList);
+    }
 
     
 }



Mime
View raw message