pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r906657 - in /hadoop/pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
Date Thu, 04 Feb 2010 21:29:16 GMT
Author: hashutosh
Date: Thu Feb  4 21:29:16 2010
New Revision: 906657

URL: http://svn.apache.org/viewvc?rev=906657&view=rev
Log:
PIG-1209: Port POJoinPackage to proactively spill

Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=906657&r1=906656&r2=906657&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Feb  4 21:29:16 2010
@@ -24,6 +24,8 @@
 
 IMPROVEMENTS
 
+PIG-1209: Port POJoinPackage to proactively spill (ashutoshc)
+
 PIG-1190: Handling of quoted strings in pig-latin/grunt commands (ashutoshc)
 
 PIG-1214: Pig 0.6 Docs fixes (chandec via olgan)

Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java?rev=906657&r1=906656&r2=906657&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
Thu Feb  4 21:29:16 2010
@@ -20,12 +20,14 @@
 import java.util.List;
 
 import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataType;
+import org.apache.pig.data.InternalCachedBag;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.io.NullableTuple;
 import org.apache.pig.impl.plan.NodeIdGenerator;
@@ -124,9 +126,17 @@
         {
             lastInputTuple = false;
             //Put n-1 inputs into bags
+            String bagType = null;
+            if (PigMapReduce.sJobConf != null) {
+                   bagType = PigMapReduce.sJobConf.get("pig.cachedbag.type");           
       
+               }
             dbs = new DataBag[numInputs];
             for (int i = 0; i < numInputs; i++) {
-                dbs[i] = mBagFactory.newDefaultBag();
+                if (bagType != null && bagType.equalsIgnoreCase("default")) {   
              
+                    dbs[i] = mBagFactory.newDefaultBag();                    
+                } else {
+                 dbs[i] = new InternalCachedBag(numInputs);
+             }    
             }
             
             //For each Nullable tuple in the input, put it



Mime
View raw message