Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: mapreduce-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1133175 - in /hadoop/mapreduce/trunk: CHANGES.txt
 src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
 src/test/mapred/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
Date: Tue, 07 Jun 2011 22:06:28 -0000
To: mapreduce-commits@hadoop.apache.org
From: todd@apache.org
Message-Id: <20110607220628.BB36023888E4@eris.apache.org>

Author: todd
Date: Tue Jun  7 22:06:28 2011
New Revision: 1133175

URL: http://svn.apache.org/viewvc?rev=1133175&view=rev
Log:
MAPREDUCE-2571. CombineFileInputFormat.getSplits throws a java.lang.ArrayStoreException. Contributed by Bochun Bai.

Added:
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1133175&r1=1133174&r2=1133175&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Jun  7 22:06:28 2011
@@ -847,6 +847,9 @@ Release 0.22.0 - Unreleased
     MAPREDUCE-2185. Fix infinite loop at creating splits using
     CombineFileInputFormat. (Ramkumar Vadali via schen)
 
+    MAPREDUCE-2571. CombineFileInputFormat.getSplits throws a
+    java.lang.ArrayStoreException. (Bochun Bai via todd)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java?rev=1133175&r1=1133174&r2=1133175&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java Tue Jun  7 22:06:28 2011
@@ -68,7 +68,17 @@ public abstract class CombineFileInputFo
 
   public InputSplit[] getSplits(JobConf job, int numSplits) 
     throws IOException {
-    return super.getSplits(new Job(job)).toArray(new InputSplit[0]);
+    List<org.apache.hadoop.mapreduce.InputSplit> newStyleSplits =
+      super.getSplits(new Job(job));
+    InputSplit[] ret = new InputSplit[newStyleSplits.size()];
+    for(int pos = 0; pos < newStyleSplits.size(); ++pos) {
+      org.apache.hadoop.mapreduce.lib.input.CombineFileSplit newStyleSplit = 
+        (org.apache.hadoop.mapreduce.lib.input.CombineFileSplit) newStyleSplits.get(pos);
+      ret[pos] = new CombineFileSplit(job, newStyleSplit.getPaths(),
+        newStyleSplit.getStartOffsets(), newStyleSplit.getLengths(),
+        newStyleSplit.getLocations());
+    }
+    return ret;
   }
   
   /**

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCombineFileInputFormat.java?rev=1133175&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCombineFileInputFormat.java (added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCombineFileInputFormat.java Tue Jun  7 22:06:28 2011
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
+import org.apache.hadoop.mapred.lib.CombineFileSplit;
+import org.apache.hadoop.mapred.lib.CombineFileRecordReader;
+
+import org.junit.Test;
+import static junit.framework.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class TestCombineFileInputFormat {
+  private static final Log LOG =
+    LogFactory.getLog(TestCombineFileInputFormat.class.getName());
+  
+  private static JobConf defaultConf = new JobConf();
+  private static FileSystem localFs = null; 
+  static {
+    try {
+      defaultConf.set("fs.default.name", "file:///");
+      localFs = FileSystem.getLocal(defaultConf);
+    } catch (IOException e) {
+      throw new RuntimeException("init failure", e);
+    }
+  }
+  private static Path workDir =
+    new Path(new Path(System.getProperty("test.build.data", "/tmp")),
+             "TestCombineFileInputFormat").makeQualified(localFs);
+
+  private static void writeFile(FileSystem fs, Path name, 
+                                String contents) throws IOException {
+    OutputStream stm;
+    stm = fs.create(name);
+    stm.write(contents.getBytes());
+    stm.close();
+  }
+  
+  /**
+   * Test getSplits
+   */
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testSplits() throws IOException {
+    JobConf job = new JobConf(defaultConf);
+    localFs.delete(workDir, true);
+    writeFile(localFs, new Path(workDir, "test.txt"), 
+              "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
+    FileInputFormat.setInputPaths(job, workDir);
+    CombineFileInputFormat format = new CombineFileInputFormat() {
+      @Override
+      public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
+        return new CombineFileRecordReader(job, (CombineFileSplit)split, reporter, CombineFileRecordReader.class);
+      }
+    };
+    final int SIZE_SPLITS = 1;
+    LOG.info("Trying to getSplits with splits = " + SIZE_SPLITS);
+    InputSplit[] splits = format.getSplits(job, SIZE_SPLITS);
+    LOG.info("Got getSplits = " + splits.length);
+    assertEquals("splits == " + SIZE_SPLITS, SIZE_SPLITS, splits.length);
+  }
+}