hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r668861 - in /hadoop/core/branches/branch-0.17: CHANGES.txt src/java/org/apache/hadoop/mapred/MapTask.java
Date Tue, 17 Jun 2008 22:39:00 GMT
Author: cdouglas
Date: Tue Jun 17 15:38:59 2008
New Revision: 668861

URL: http://svn.apache.org/viewvc?rev=668861&view=rev
Log:
HADOOP-3475. Fix MapTask to correctly size the accounting allocation of
io.sort.mb.



Modified:
    hadoop/core/branches/branch-0.17/CHANGES.txt
    hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java

Modified: hadoop/core/branches/branch-0.17/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=668861&r1=668860&r2=668861&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.17/CHANGES.txt Tue Jun 17 15:38:59 2008
@@ -17,6 +17,8 @@
     HADOOP-3477. Fix build to not package contrib/*/bin twice in
     distributions.  (Adam Heath via cutting)
 
+    HADOOP-3475. Fix MapTask to correctly size the accounting allocation of
+    io.sort.mb. (cdouglas)
 
 Release 0.17.1 - Unreleased
 

Modified: hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java?rev=668861&r1=668860&r2=668861&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/MapTask.java Tue Jun
17 15:38:59 2008
@@ -310,6 +310,9 @@
     private static final int KEYSTART = 1;  // key offset in acct
     private static final int VALSTART = 2;  // val offset in acct
     private static final int ACCTSIZE = 3;  // total #fields in acct
+    private static final int RECSIZE =
+                       (ACCTSIZE + 1) * 4;  // acct bytes per record
+
 
     // spill accounting
     private volatile int numSpills = 0;
@@ -352,12 +355,12 @@
       // buffers and accounting
       int maxMemUsage = sortmb << 20;
       int recordCapacity = (int)(maxMemUsage * recper);
-      recordCapacity += (recordCapacity >>> 2) % 4;
+      recordCapacity -= recordCapacity % RECSIZE;
       kvbuffer = new byte[maxMemUsage - recordCapacity];
       bufvoid = kvbuffer.length;
-      int kvcapacity = recordCapacity >>> 2;
-      kvoffsets = new int[kvcapacity];
-      kvindices = new int[recordCapacity - kvcapacity];
+      recordCapacity /= RECSIZE;
+      kvoffsets = new int[recordCapacity];
+      kvindices = new int[recordCapacity * ACCTSIZE];
       softBufferLimit = (int)(kvbuffer.length * spillper);
       softRecordLimit = (int)(kvoffsets.length * spillper);
       // k/v serialization



Mime
View raw message