pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1704996 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/ test/org/apache/pig/test/
Date Thu, 24 Sep 2015 05:45:43 GMT
Author: daijy
Date: Thu Sep 24 05:45:43 2015
New Revision: 1704996

URL: http://svn.apache.org/viewvc?rev=1704996&view=rev
Log:
PIG-4683: Nested order is broken after PIG-3591 in some cases

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
    pig/trunk/test/org/apache/pig/test/TestSecondarySort.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1704996&r1=1704995&r2=1704996&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Sep 24 05:45:43 2015
@@ -47,6 +47,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4683: Nested order is broken after PIG-3591 in some cases (daijy)
+
 PIG-4679: Performance degradation due to InputSizeReducerEstimator since PIG-3754 (daijy)
 
 PIG-4315: MergeJoin or Split followed by order by gives NPE in Tez (rohini)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java?rev=1704996&r1=1704995&r2=1704996&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
Thu Sep 24 05:45:43 2015
@@ -257,8 +257,15 @@ public class POPackage extends PhysicalO
                     NullableTuple ntup = tupIter.next();
                     int index = ntup.getIndex();
                     if (index == numInputs - 1) {
-                        dbs[index] = new PeekedBag(pkgr, ntup, tupIter, keyWritable);
-                        break;
+                        if (pkgr.getUseSecondaryKey()) {
+                            if (dbs[index] == null) {
+                                dbs[index] = useDefaultBag ? BagFactory.getInstance()
+                                        .newDefaultBag() : new InternalCachedBag(numInputs);
+                            }
+                        } else {
+                            dbs[index] = new PeekedBag(pkgr, ntup, tupIter, keyWritable);
+                            break;
+                        }
                     }
                     Tuple copy = pkgr.getValueTuple(keyWritable, ntup, index);
 

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java?rev=1704996&r1=1704995&r2=1704996&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
Thu Sep 24 05:45:43 2015
@@ -353,6 +353,9 @@ public class Packager implements Illustr
     public void setUseSecondaryKey(boolean useSecondaryKey) {
         this.useSecondaryKey = useSecondaryKey;
     }
+    public boolean getUseSecondaryKey() {
+        return useSecondaryKey;
+    }
 
     public void setPackageType(PackageType type) {
         this.pkgType = type;

Modified: pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=1704996&r1=1704995&r2=1704996&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Thu Sep 24 05:45:43 2015
@@ -448,6 +448,38 @@ public abstract class TestSecondarySort
 
     }
 
+    @Test
+    public void testNestedSortMultiQueryEndToEnd3() throws Exception {
+        File input1 = Util.createTempFileDelOnExit("test", "txt");
+        PrintStream ps1 = new PrintStream(new FileOutputStream(input1));
+        ps1.println("a\t0");
+        ps1.println("a\t2");
+        ps1.println("a\t1");
+        ps1.close();
+        Util.copyFromLocalToCluster(cluster, input1.getCanonicalPath(), "testNestedSortMultiQueryEndToEnd3-input-1.txt");
+
+        File input2 = Util.createTempFileDelOnExit("test", "txt");
+        PrintStream ps2 = new PrintStream(new FileOutputStream(input2));
+        ps2.println("a");
+        ps2.close();
+        Util.copyFromLocalToCluster(cluster, input2.getCanonicalPath(), "testNestedSortMultiQueryEndToEnd3-input-2.txt");
+
+        try {
+            pigServer.setBatchOn();
+            pigServer.registerQuery("a = load 'testNestedSortMultiQueryEndToEnd3-input-1.txt'
as (a0:chararray, a1:chararray);");
+            pigServer.registerQuery("b = load 'testNestedSortMultiQueryEndToEnd3-input-2.txt'
as (b0);");
+            pigServer.registerQuery("c = cogroup b by b0, a by a0;");
+            pigServer.registerQuery("d = foreach c {a_sorted = order a by a1 desc;generate
group, a_sorted, b;}");
+            Iterator<Tuple> iter = pigServer.openIterator("d");
+
+            assertEquals(iter.next().toString(), "(a,{(a,2),(a,1),(a,0)},{(a)})");
+        } finally {
+            Util.deleteFile(cluster, "testNestedSortMultiQueryEndToEnd3-input-1.txt");
+            Util.deleteFile(cluster, "testNestedSortMultiQueryEndToEnd3-input-2.txt");
+        }
+
+    }
+
     // See PIG-1978
     @Test
     public void testForEachTwoInput() throws Exception {



Mime
View raw message