pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1605947 - in /pig/branches/branch-0.13: ./ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/ src/org/apache/pig/impl/streaming/
Date Fri, 27 Jun 2014 05:10:56 GMT
Author: daijy
Date: Fri Jun 27 05:10:55 2014
New Revision: 1605947

URL: http://svn.apache.org/r1605947
Log:
PIG-4036: Fix e2e failures - JobManagement_3, CmdErrors_3 and BigData_4

Modified:
    pig/branches/branch-0.13/CHANGES.txt
    pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
    pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
    pig/branches/branch-0.13/src/org/apache/pig/impl/streaming/StreamingCommand.java

Modified: pig/branches/branch-0.13/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.13/CHANGES.txt?rev=1605947&r1=1605946&r2=1605947&view=diff
==============================================================================
--- pig/branches/branch-0.13/CHANGES.txt (original)
+++ pig/branches/branch-0.13/CHANGES.txt Fri Jun 27 05:10:55 2014
@@ -153,6 +153,8 @@ PIG-3882: Multiquery off mode execution 
  
 BUG FIXES
 
+PIG-4036: Fix e2e failures - JobManagement_3, CmdErrors_3 and BigData_4 (daijy)
+
 PIG-4037: TestHBaseStorage, TestAccumuloPigCluster has failures with hadoopversion=23 (daijy)
 
 PIG-4005: depend on hbase-hadoop2-compat rather than hbase-hadoop1-compat when hbaseversion
is 95 (daijy)

Modified: pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java?rev=1605947&r1=1605946&r2=1605947&view=diff
==============================================================================
--- pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
(original)
+++ pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
Fri Jun 27 05:10:55 2014
@@ -84,6 +84,8 @@ public class POPackage extends PhysicalO
 
     private boolean useDefaultBag = false;
 
+    private boolean lastBagReadOnly = true;
+
     protected Packager pkgr;
 
     private PigNullableWritable keyWritable;
@@ -194,6 +196,12 @@ public class POPackage extends PhysicalO
                     useDefaultBag = true;
                 }
             }
+            // If multiquery, the last bag is InternalCachedBag and should not
+            // set ReadOnly flag, otherwise we will materialize again to another
+            // InternalCachedBag
+            if (pkgr instanceof MultiQueryPackager) {
+                lastBagReadOnly = false;
+            }
         }
         int numInputs = pkgr.getNumInputs(keyWritable.getIndex());
         boolean[] readOnce = new boolean[numInputs];
@@ -216,7 +224,10 @@ public class POPackage extends PhysicalO
                 }
 
             } else {
-                readOnce[numInputs - 1] = true;
+                if (lastBagReadOnly) {
+                    readOnce[numInputs - 1] = true;
+                }
+
                 // We know the tuples will come sorted by index, so we can wrap
                 // the last input in a ReadOnceBag and let the Packager decide
                 // whether or not to read into memory

Modified: pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java?rev=1605947&r1=1605946&r2=1605947&view=diff
==============================================================================
--- pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
(original)
+++ pig/branches/branch-0.13/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
Fri Jun 27 05:10:55 2014
@@ -481,6 +481,6 @@ public class Packager implements Illustr
                 // POJoinPackage in the pipeline and is also blocking the pipeline;
                 // constructor argument should be 2 * numInputs. But for one obscure
                 // case we don't want to pay the penalty all the time.
-                : new InternalCachedBag(numInputs-1);
+                : new InternalCachedBag(numInputs);
     }
 }

Modified: pig/branches/branch-0.13/src/org/apache/pig/impl/streaming/StreamingCommand.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.13/src/org/apache/pig/impl/streaming/StreamingCommand.java?rev=1605947&r1=1605946&r2=1605947&view=diff
==============================================================================
--- pig/branches/branch-0.13/src/org/apache/pig/impl/streaming/StreamingCommand.java (original)
+++ pig/branches/branch-0.13/src/org/apache/pig/impl/streaming/StreamingCommand.java Fri Jun
27 05:10:55 2014
@@ -34,7 +34,7 @@ import org.apache.pig.builtin.PigStorage
 import org.apache.pig.builtin.PigStreaming;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.zookeeper.Shell;
+import org.apache.hadoop.util.Shell;
 
 
 /**



Mime
View raw message