hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r648519 - in /hadoop/core/branches/branch-0.17: CHANGES.txt src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
Date Wed, 16 Apr 2008 05:12:43 GMT
Author: acmurthy
Date: Tue Apr 15 22:12:42 2008
New Revision: 648519

URL: http://svn.apache.org/viewvc?rev=648519&view=rev
Log:
Merge -r 648515:648516 from trunk to branch-0.17 to fix HADOOP-3242

Modified:
    hadoop/core/branches/branch-0.17/CHANGES.txt
    hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java
    hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java

Modified: hadoop/core/branches/branch-0.17/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=648519&r1=648518&r2=648519&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.17/CHANGES.txt Tue Apr 15 22:12:42 2008
@@ -572,6 +572,10 @@
     change maxentToSameCluster to run the correct number of jobs.
     (Runping Qi via cdouglas)
 
+    HADOOP-3242. Fix the RecordReader of SequenceFileAsBinaryInputFormat to
+    correctly read from the start of the split and not the beginning of the
+    file. (cdouglas via acmurthy) 
+
 Release 0.16.3 - 2008-04-16
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java?rev=648519&r1=648518&r2=648519&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java
Tue Apr 15 22:12:42 2008
@@ -66,7 +66,11 @@
       FileSystem fs = path.getFileSystem(conf);
       this.in = new SequenceFile.Reader(fs, path, conf);
       this.end = split.getStart() + split.getLength();
+      if (split.getStart() > in.getPosition())
+        in.sync(split.getStart());                  // sync to start
+      this.start = in.getPosition();
       vbytes = in.createValueBytes();
+      done = start >= end;
     }
 
     public BytesWritable createKey() {

Modified: hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java?rev=648519&r1=648518&r2=648519&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
(original)
+++ hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
Tue Apr 15 22:12:42 2008
@@ -69,30 +69,32 @@
     Text cmpkey = new Text();
     Text cmpval = new Text();
     DataInputBuffer buf = new DataInputBuffer();
-    RecordReader<BytesWritable,BytesWritable> reader =
-      bformat.getRecordReader(new FileSplit(file, 0,
-                              fs.getFileStatus(file).getLen(), 
-                              (String[])null), job, Reporter.NULL);
-    try {
-      while (reader.next(bkey, bval)) {
-        tkey.set(Integer.toString(r.nextInt(), 36));
-        tval.set(Long.toString(r.nextLong(), 36));
-        buf.reset(bkey.get(), bkey.getSize());
-        cmpkey.readFields(buf);
-        buf.reset(bval.get(), bval.getSize());
-        cmpval.readFields(buf);
-        assertTrue(
-            "Keys don't match: " + "*" + cmpkey.toString() + ":" +
-                                         tkey.toString() + "*",
-            cmpkey.toString().equals(tkey.toString()));
-        assertTrue(
-            "Vals don't match: " + "*" + cmpval.toString() + ":" +
-                                         tval.toString() + "*",
-            cmpval.toString().equals(tval.toString()));
-        ++count;
+    final int NUM_SPLITS = 3;
+    job.setInputPath(file);
+    for (InputSplit split : bformat.getSplits(job, NUM_SPLITS)) {
+      RecordReader<BytesWritable,BytesWritable> reader =
+        bformat.getRecordReader(split, job, Reporter.NULL);
+      try {
+        while (reader.next(bkey, bval)) {
+          tkey.set(Integer.toString(r.nextInt(), 36));
+          tval.set(Long.toString(r.nextLong(), 36));
+          buf.reset(bkey.get(), bkey.getSize());
+          cmpkey.readFields(buf);
+          buf.reset(bval.get(), bval.getSize());
+          cmpval.readFields(buf);
+          assertTrue(
+              "Keys don't match: " + "*" + cmpkey.toString() + ":" +
+                                           tkey.toString() + "*",
+              cmpkey.toString().equals(tkey.toString()));
+          assertTrue(
+              "Vals don't match: " + "*" + cmpval.toString() + ":" +
+                                           tval.toString() + "*",
+              cmpval.toString().equals(tval.toString()));
+          ++count;
+        }
+      } finally {
+        reader.close();
       }
-    } finally {
-      reader.close();
     }
     assertEquals("Some records not found", RECORDS, count);
   }



Mime
View raw message