hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r615300 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/MultiFileInputFormat.java
Date Fri, 25 Jan 2008 18:28:53 GMT
Author: nigel
Date: Fri Jan 25 10:28:53 2008
New Revision: 615300

URL: http://svn.apache.org/viewvc?rev=615300&view=rev
Log:
HADOOP-2640. Fix a bug in MultiFileSplitInputFormat that was always returning 1 split in some
circumstances. Contributed by Enis Soztutar.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MultiFileInputFormat.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=615300&r1=615299&r2=615300&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Jan 25 10:28:53 2008
@@ -564,6 +564,9 @@
     HADOOP-1195. All code in FSNamesystem checks the return value
     of getDataNode for null before using it. (dhruba)
 
+    HADOOP-2640. Fix a bug in MultiFileSplitInputFormat that was always
+    returning 1 split in some circumstances. (Enis Soztutar via nigel)
+
 Release 0.15.3 - 2008-01-18
 
   BUG FIXES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MultiFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MultiFileInputFormat.java?rev=615300&r1=615299&r2=615300&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MultiFileInputFormat.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MultiFileInputFormat.java Fri Jan
25 10:28:53 2008
@@ -55,7 +55,7 @@
         lengths[i] = fs.getContentLength(paths[i]);
         totLength += lengths[i];
       }
-      float avgLengthPerSplit = ((float)totLength) / numSplits;
+      double avgLengthPerSplit = ((double)totLength) / numSplits;
       long cumulativeLength = 0;
 
       int startIndex = 0;
@@ -80,14 +80,14 @@
     return splits.toArray(new MultiFileSplit[splits.size()]);    
   }
 
-  private int findSize(int splitIndex, float avgLengthPerSplit
+  private int findSize(int splitIndex, double avgLengthPerSplit
       , long cumulativeLength , int startIndex, long[] lengths) {
     
     if(splitIndex == lengths.length - 1)
       return lengths.length - startIndex;
     
     long goalLength = (long)((splitIndex + 1) * avgLengthPerSplit);
-    int partialLength = 0;
+    long partialLength = 0;
     // accumulate till just above the goal length;
     for(int i = startIndex; i < lengths.length; i++) {
       partialLength += lengths[i];
@@ -98,6 +98,7 @@
     return lengths.length - startIndex;
   }
   
+  @Override
   public abstract RecordReader<K, V> getRecordReader(InputSplit split,
       JobConf job, Reporter reporter)
       throws IOException;



Mime
View raw message