Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 31802 invoked from network); 16 Apr 2008 05:13:17 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 16 Apr 2008 05:13:17 -0000 Received: (qmail 52378 invoked by uid 500); 16 Apr 2008 05:13:18 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 52248 invoked by uid 500); 16 Apr 2008 05:13:18 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 52239 invoked by uid 99); 16 Apr 2008 05:13:18 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Apr 2008 22:13:18 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Apr 2008 05:12:32 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D38351A9832; Tue, 15 Apr 2008 22:12:44 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r648519 - in /hadoop/core/branches/branch-0.17: CHANGES.txt src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java Date: Wed, 16 Apr 2008 05:12:43 -0000 To: core-commits@hadoop.apache.org From: acmurthy@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080416051244.D38351A9832@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: acmurthy Date: Tue Apr 15 22:12:42 2008 New Revision: 648519 URL: http://svn.apache.org/viewvc?rev=648519&view=rev Log: Merge -r 648515:648516 from trunk to branch-0.17 to fix HADOOP-3242 Modified: hadoop/core/branches/branch-0.17/CHANGES.txt hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java Modified: hadoop/core/branches/branch-0.17/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/CHANGES.txt?rev=648519&r1=648518&r2=648519&view=diff ============================================================================== --- hadoop/core/branches/branch-0.17/CHANGES.txt (original) +++ hadoop/core/branches/branch-0.17/CHANGES.txt Tue Apr 15 22:12:42 2008 @@ -572,6 +572,10 @@ change maxentToSameCluster to run the correct number of jobs. (Runping Qi via cdouglas) + HADOOP-3242. Fix the RecordReader of SequenceFileAsBinaryInputFormat to + correctly read from the start of the split and not the beginning of the + file. (cdouglas via acmurthy) + Release 0.16.3 - 2008-04-16 BUG FIXES Modified: hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java?rev=648519&r1=648518&r2=648519&view=diff ============================================================================== --- hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java (original) +++ hadoop/core/branches/branch-0.17/src/java/org/apache/hadoop/mapred/SequenceFileAsBinaryInputFormat.java Tue Apr 15 22:12:42 2008 @@ -66,7 +66,11 @@ FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); + if (split.getStart() > in.getPosition()) + in.sync(split.getStart()); // sync to start + this.start = in.getPosition(); vbytes = in.createValueBytes(); + done = start >= end; } public BytesWritable createKey() { Modified: hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java?rev=648519&r1=648518&r2=648519&view=diff ============================================================================== --- hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java (original) +++ hadoop/core/branches/branch-0.17/src/test/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java Tue Apr 15 22:12:42 2008 @@ -69,30 +69,32 @@ Text cmpkey = new Text(); Text cmpval = new Text(); DataInputBuffer buf = new DataInputBuffer(); - RecordReader reader = - bformat.getRecordReader(new FileSplit(file, 0, - fs.getFileStatus(file).getLen(), - (String[])null), job, Reporter.NULL); - try { - while (reader.next(bkey, bval)) { - tkey.set(Integer.toString(r.nextInt(), 36)); - tval.set(Long.toString(r.nextLong(), 36)); - buf.reset(bkey.get(), bkey.getSize()); - cmpkey.readFields(buf); - buf.reset(bval.get(), bval.getSize()); - cmpval.readFields(buf); - assertTrue( - "Keys don't match: " + "*" + cmpkey.toString() + ":" + - tkey.toString() + "*", - cmpkey.toString().equals(tkey.toString())); - assertTrue( - "Vals don't match: " + "*" + cmpval.toString() + ":" + - tval.toString() + "*", - cmpval.toString().equals(tval.toString())); - ++count; + final int NUM_SPLITS = 3; + job.setInputPath(file); + for (InputSplit split : bformat.getSplits(job, NUM_SPLITS)) { + RecordReader reader = + bformat.getRecordReader(split, job, Reporter.NULL); + try { + while (reader.next(bkey, bval)) { + tkey.set(Integer.toString(r.nextInt(), 36)); + tval.set(Long.toString(r.nextLong(), 36)); + buf.reset(bkey.get(), bkey.getSize()); + cmpkey.readFields(buf); + buf.reset(bval.get(), bval.getSize()); + cmpval.readFields(buf); + assertTrue( + "Keys don't match: " + "*" + cmpkey.toString() + ":" + + tkey.toString() + "*", + cmpkey.toString().equals(tkey.toString())); + assertTrue( + "Vals don't match: " + "*" + cmpval.toString() + ":" + + tval.toString() + "*", + cmpval.toString().equals(tval.toString())); + ++count; + } + } finally { + reader.close(); } - } finally { - reader.close(); } assertEquals("Some records not found", RECORDS, count); }