pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1580091 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java test/org/apache/pig/test/TestSecondarySort.java
Date Sat, 22 Mar 2014 00:10:38 GMT
Author: daijy
Date: Sat Mar 22 00:10:38 2014
New Revision: 1580091

URL: http://svn.apache.org/r1580091
Log:
PIG-3827: Custom partitioner is not picked up with secondary sort optimization

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
    pig/trunk/test/org/apache/pig/test/TestSecondarySort.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1580091&r1=1580090&r2=1580091&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Mar 22 00:10:38 2014
@@ -257,6 +257,8 @@ PIG-3480: TFile-based tmpfile compressio
 
 BUG FIXES
 
+PIG-3827: Custom partitioner is not picked up with secondary sort optimization (daijy)
+
 PIG-3826: Outer join with PushDownForEachFlatten generates wrong result (daijy)
 
 PIG-3820: TestAvroStorage fail on some OS (daijy)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java?rev=1580091&r1=1580090&r2=1580091&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
Sat Mar 22 00:10:38 2014
@@ -152,6 +152,10 @@ public class SecondaryKeyOptimizer exten
         if (mr.isGlobalSort())
             return;
 
+        // Don't optimize when we already have a custom partitioner
+        if (mr.getCustomPartitioner()!=null)
+            return;
+
         List<PhysicalOperator> mapLeaves = mr.mapPlan.getLeaves();
         if (mapLeaves == null || mapLeaves.size() != 1) {
             log

Modified: pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=1580091&r1=1580090&r2=1580091&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Sat Mar 22 00:10:38 2014
@@ -455,5 +455,36 @@ public class TestSecondarySort {
 
         Util.deleteFile(cluster, clusterFilePath);
     }
+    
+    @Test
+    // Once custom partitioner is used, we cannot use secondary key optimizer, see PIG-3827
+    public void testCustomPartitionerWithSort() throws Exception {
+        File tmpFile1 = Util.createTempFileDelOnExit("test", "txt");
+        PrintStream ps1 = new PrintStream(new FileOutputStream(tmpFile1));
+        ps1.println("1\t2\t3");
+        ps1.println("1\t3\t4");
+        ps1.println("1\t4\t4");
+        ps1.println("1\t2\t4");
+        ps1.println("1\t8\t4");
+        ps1.println("2\t3\t4");
+        ps1.close();
+
+        String clusterPath = Util.removeColon(tmpFile1.getCanonicalPath());
+
+        Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), clusterPath);
+        pigServer.registerQuery("A = LOAD '" + Util.encodeEscape(clusterPath) + "' AS (a0,
a1, a2);");
+        pigServer.registerQuery("B = group A by $0 PARTITION BY org.apache.pig.test.utils.WrongCustomPartitioner
parallel 2;");
+        pigServer.registerQuery("C = foreach B { D = order A by a1 desc; generate group,
D;};");
+        boolean captureException = false;
+        try {
+            pigServer.openIterator("C");
+        } catch (Exception e) {
+            captureException = true;
+        }
+        
+        assertTrue(captureException);
+        
+        Util.deleteFile(cluster, clusterPath);
+    }
 }
 



Mime
View raw message