pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r940595 - in /hadoop/pig/branches/branch-0.7: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
Date Mon, 03 May 2010 20:13:32 GMT
Author: daijy
Date: Mon May  3 20:13:31 2010
New Revision: 940595

URL: http://svn.apache.org/viewvc?rev=940595&view=rev
Log:
PIG-1394: POCombinerPackage hold too much memory for InternalCachedBag

Modified:
    hadoop/pig/branches/branch-0.7/CHANGES.txt
    hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java

Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=940595&r1=940594&r2=940595&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/CHANGES.txt Mon May  3 20:13:31 2010
@@ -187,6 +187,8 @@ OPTIMIZATIONS
 
 BUG FIXES
 
+PIG-1394: POCombinerPackage hold too much memory for InternalCachedBag (daijy)
+
 PIG-1303: Inconsistent instantiation of parametrized UDFs (jrussek and dvryaboy)
 
 PIG-1348: PigStorage making unnecessary byte array copy when storing data

Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java?rev=940595&r1=940594&r2=940595&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
(original)
+++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
Mon May  3 20:13:31 2010
@@ -66,7 +66,8 @@ public class POCombinerPackage extends P
     private boolean[] keyPositions;
     
     private Map<Integer, Integer> keyLookup;
-                             
+    
+    private int numBags;
 
     /**
      * A new POPostCombinePackage will be constructed as a near clone of the
@@ -91,6 +92,10 @@ public class POCombinerPackage extends P
         if (bags != null) {
             mBags = Arrays.copyOf(bags, bags.length);
         }
+        numBags = 0;
+        for (int i = 0; i < mBags.length; i++) {
+            if (mBags[i]) numBags++;            
+        }
         if (keyPos != null) {
             keyPositions = Arrays.copyOf(keyPos, keyPos.length);
         }
@@ -122,7 +127,7 @@ public class POCombinerPackage extends P
         keyLookup = lrKeyInfo.second;
     }
 
-    private DataBag createDataBag() {
+    private DataBag createDataBag(int numBags) {
     	String bagType = null;
         if (PigMapReduce.sJobConf != null) {
    			bagType = PigMapReduce.sJobConf.get("pig.cachedbag.type");       			
@@ -131,7 +136,7 @@ public class POCombinerPackage extends P
     	if (bagType != null && bagType.equalsIgnoreCase("default")) {
     		return new NonSpillableDataBag();
     	}
-    	return new InternalCachedBag();  	
+    	return new InternalCachedBag(numBags);  	
     }
     
     @Override
@@ -140,7 +145,7 @@ public class POCombinerPackage extends P
         //Create numInputs bags
         Object[] fields = new Object[mBags.length];
         for (int i = 0; i < mBags.length; i++) {
-            if (mBags[i]) fields[i] = createDataBag();            
+            if (mBags[i]) fields[i] = createDataBag(numBags);            
         }
         
         // For each indexed tup in the inp, split them up and place their



Mime
View raw message