hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r651009 - in /hadoop/core/branches/branch-0.17: CHANGES.txt src/java/org/apache/hadoop/fs/BlockLocation.java src/java/org/apache/hadoop/mapred/FileInputFormat.java src/test/org/apache/hadoop/mapred/TestFileInputFormat.java
Date Wed, 23 Apr 2008 19:07:22 GMT
Author: omalley
Date: Wed Apr 23 12:07:17 2008
New Revision: 651009

URL: http://svn.apache.org/viewvc?rev=651009&view=rev
Log:
Merge of -r 651005:651006 from trunk to 0.17 to fix HADOOP-3285.

Added:
    hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestFileInputFormat.java
      - copied unchanged from r651006, hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestFileInputFormat.java
Modified:
    hadoop/core/branches/branch-0.17/CHANGES.txt
    hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/fs/BlockLocation.java
    hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/FileInputFormat.java

Modified: hadoop/core/branches/branch-0.17/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=651009&r1=651008&r2=651009&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.17/CHANGES.txt Wed Apr 23 12:07:17 2008
@@ -577,6 +577,9 @@
     HADOOP-3286. Prevent collisions in gridmix output dirs by increasing the
     granularity of the timestamp. (Runping Qi via cdouglas)
 
+    HADOOP-3285. Fix input split locality when the splits align to
+    fs blocks. (omalley)
+
 Release 0.16.3 - 2008-04-16
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/fs/BlockLocation.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/fs/BlockLocation.java?rev=651009&r1=651008&r2=651009&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/fs/BlockLocation.java (original)
+++ hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/fs/BlockLocation.java Wed
Apr 23 12:07:17 2008
@@ -177,4 +177,16 @@
       hosts[i] = host.toString();
     }
   }
+  
+  public String toString() {
+    StringBuilder result = new StringBuilder();
+    result.append(offset);
+    result.append(',');
+    result.append(length);
+    for(String h: hosts) {
+      result.append(',');
+      result.append(h);
+    }
+    return result.toString();
+  }
 }

Modified: hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/FileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/FileInputFormat.java?rev=651009&r1=651008&r2=651009&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/FileInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/FileInputFormat.java
Wed Apr 23 12:07:17 2008
@@ -279,12 +279,17 @@
   protected int getBlockIndex(BlockLocation[] blkLocations, 
                               long offset) {
     for (int i = 0 ; i < blkLocations.length; i++) {
+      // is the offset inside this block?
       if ((blkLocations[i].getOffset() <= offset) &&
-        ((blkLocations[i].getOffset() + blkLocations[i].getLength()) >= 
-        offset))
-          return i;
+          (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())){
+        return i;
+      }
     }
-    return 0;
+    BlockLocation last = blkLocations[blkLocations.length];
+    long fileLength = last.getOffset() + last.getLength() -1;
+    throw new IllegalArgumentException("Offset " + offset + 
+                                       " is outside of file (0.." +
+                                       fileLength + ")");
   }
 
   /**



Mime
View raw message