hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r669472 - in /hadoop/core/trunk: CHANGES.txt src/core/org/apache/hadoop/fs/HarFileSystem.java src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
Date Thu, 19 Jun 2008 12:46:16 GMT
Author: ddas
Date: Thu Jun 19 05:46:15 2008
New Revision: 669472

URL: http://svn.apache.org/viewvc?rev=669472&view=rev
Log:
HADOOP-3580. Fixes a problem to do with specifying a har as an input to a job. Contributed
by Mahadev Konar.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=669472&r1=669471&r2=669472&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jun 19 05:46:15 2008
@@ -617,6 +617,9 @@
     HADOOP-3533. Add deprecated methods to provide API compatibility
     between 0.18 and 0.17. Remove the deprecated methods in trunk. (omalley)
 
+    HADOOP-3580. Fixes a problem to do with specifying a har as an input to 
+    a job. (Mahadev Konar via ddas)
+
 Release 0.17.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java?rev=669472&r1=669471&r2=669472&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/HarFileSystem.java Thu Jun 19 05:46:15
2008
@@ -343,8 +343,24 @@
     }
     FileStatus fsFile = fs.getFileStatus(new Path(archivePath,
         harStatus.getPartName()));
-    return fs.getFileBlockLocations(fsFile, 
-        harStatus.getStartIndex(), harStatus.getLength());
+    BlockLocation[] rawBlocks = fs.getFileBlockLocations(fsFile, 
+        harStatus.getStartIndex() + start, len);
+    return fakeBlockLocations(rawBlocks, harStatus.getStartIndex());
+  }
+  
+  /**
+   * fake the rawblocks since map reduce uses the block offsets to 
+   * fo some computations regarding the blocks
+   * @param rawBlocks the raw blocks returned by the filesystem
+   * @return faked blocks with changed offsets.
+   */
+  private BlockLocation[] fakeBlockLocations(BlockLocation[] rawBlocks, 
+		  long startIndex) {
+	for (BlockLocation block : rawBlocks) {
+		long rawOffset = block.getOffset();
+		block.setOffset(rawOffset - startIndex);
+	}
+	return rawBlocks;
   }
   
   /**

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java?rev=669472&r1=669471&r2=669472&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/FileInputFormat.java Thu Jun 19
05:46:15 2008
@@ -331,7 +331,7 @@
         return i;
       }
     }
-    BlockLocation last = blkLocations[blkLocations.length];
+    BlockLocation last = blkLocations[blkLocations.length -1];
     long fileLength = last.getOffset() + last.getLength() -1;
     throw new IllegalArgumentException("Offset " + offset + 
                                        " is outside of file (0.." +



Mime
View raw message