Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0C8169835 for ; Mon, 5 Mar 2012 23:57:11 +0000 (UTC) Received: (qmail 12085 invoked by uid 500); 5 Mar 2012 23:57:10 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 12043 invoked by uid 500); 5 Mar 2012 23:57:10 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 12035 invoked by uid 99); 5 Mar 2012 23:57:10 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 05 Mar 2012 23:57:10 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 05 Mar 2012 23:57:07 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B66612388CA2; Mon, 5 Mar 2012 23:56:46 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1297293 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/jav... Date: Mon, 05 Mar 2012 23:56:46 -0000 To: mapreduce-commits@hadoop.apache.org From: tucu@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120305235646.B66612388CA2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: tucu Date: Mon Mar 5 23:56:46 2012 New Revision: 1297293 URL: http://svn.apache.org/viewvc?rev=1297293&view=rev Log: MAPREDUCE-3952. In MR2, when Total input paths to process == 1, CombinefileInputFormat.getSplits() returns 0 split. (zhenxiao via tucu) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1297293&r1=1297292&r2=1297293&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Mar 5 23:56:46 2012 @@ -128,6 +128,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3728. ShuffleHandler can't access results when configured in a secure mode (ahmed via tucu) + MAPREDUCE-3952. In MR2, when Total input paths to process == 1, + CombinefileInputFormat.getSplits() returns 0 split. (zhenxiao via tucu) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java?rev=1297293&r1=1297292&r2=1297293&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java Mon Mar 5 23:56:46 2012 @@ -514,7 +514,7 @@ public abstract class CombineFileInputFo long left = locations[i].getLength(); long myOffset = locations[i].getOffset(); long myLength = 0; - while (left > 0) { + do { if (maxSize == 0) { myLength = left; } else { @@ -536,7 +536,7 @@ public abstract class CombineFileInputFo myOffset += myLength; blocksList.add(oneblock); - } + } while (left > 0); } blocks = blocksList.toArray(new OneBlockInfo[blocksList.size()]); } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java?rev=1297293&r1=1297292&r2=1297293&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java Mon Mar 5 23:56:46 2012 @@ -42,6 +42,7 @@ import org.apache.hadoop.mapreduce.TaskA import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.junit.Test; public class TestCombineFileInputFormat extends TestCase { @@ -1111,6 +1112,34 @@ public class TestCombineFileInputFormat } } } + + /** + * Test when the input file's length is 0. + */ + @Test + public void testForEmptyFile() throws Exception { + Configuration conf = new Configuration(); + FileSystem fileSys = FileSystem.get(conf); + Path file = new Path("test" + "/file"); + FSDataOutputStream out = fileSys.create(file, true, + conf.getInt("io.file.buffer.size", 4096), (short) 1, (long) BLOCKSIZE); + out.write(new byte[0]); + out.close(); + + // split it using a CombinedFile input format + DummyInputFormat inFormat = new DummyInputFormat(); + Job job = Job.getInstance(conf); + FileInputFormat.setInputPaths(job, "test"); + List splits = inFormat.getSplits(job); + assertEquals(splits.size(), 1); + CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0); + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(file.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(0, fileSplit.getLength(0)); + + fileSys.delete(file.getParent(), true); + } static class TestFilter implements PathFilter { private Path p;