hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject git commit: MAPREDUCE-6147. Support mapreduce.input.fileinputformat.split.maxsize. (Zhihai Xu via kasha)
Date Mon, 03 Nov 2014 23:55:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-1 689ca40fc -> ebec913e2


MAPREDUCE-6147. Support mapreduce.input.fileinputformat.split.maxsize. (Zhihai Xu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebec913e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebec913e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebec913e

Branch: refs/heads/branch-1
Commit: ebec913e20d0081e2e30e13817e9023565e0b920
Parents: 689ca40
Author: Karthik Kambatla <kasha@apache.org>
Authored: Mon Nov 3 15:55:22 2014 -0800
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Mon Nov 3 15:55:22 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +++
 .../mapred/lib/CombineFileInputFormat.java      | 20 ++++++++++++++++--
 .../mapred/lib/TestCombineFileInputFormat.java  | 22 ++++++++++++++++++++
 3 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebec913e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 904bbd5..a695be2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -61,6 +61,9 @@ Release 1.3.0 - unreleased
     HADOOP-10614. CBZip2InputStream is not threadsafe (Xiangrui Meng via Sandy
     Ryza)
 
+    MAPREDUCE-6147. Support mapreduce.input.fileinputformat.split.maxsize.
+    (Zhihai Xu via kasha)
+
   BUG FIXES
 
     MAPREDUCE-4490. Fixed LinuxTaskController to re-initialize user log

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebec913e/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java b/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
index 825100f..4977a35 100644
--- a/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
+++ b/src/mapred/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
@@ -141,7 +141,23 @@ public abstract class CombineFileInputFormat<K, V>
     }
     return codec instanceof SplittableCompressionCodec;
   }
-  
+
+  /**
+   * First get "mapred.max.split.size".
+   * If "mapred.max.split.size" is not set,
+   * then get "mapreduce.input.fileinputformat.split.maxsize".
+   * If "mapreduce.input.fileinputformat.split.maxsize" is not set,
+   * then return 0.
+   */
+  long getConfiguredMaxSplitSize(JobConf job) {
+    long maxSize = job.getLong("mapred.max.split.size", -1L);
+    if (maxSize == -1L) {
+      maxSize = job.getLong("mapreduce.input.fileinputformat.split.maxsize",
+          0);
+    }
+    return maxSize;
+  }
+
   /**
    * default constructor
    */
@@ -171,7 +187,7 @@ public abstract class CombineFileInputFormat<K, V>
     if (maxSplitSize != 0) {
       maxSize = maxSplitSize;
     } else {
-      maxSize = job.getLong("mapred.max.split.size", 0);
+      maxSize = getConfiguredMaxSplitSize(job);
     }
     if (minSizeNode != 0 && maxSize != 0 && minSizeNode > maxSize) {
       throw new IOException("Minimum split size pernode " + minSizeNode +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebec913e/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java b/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java
index 0d9e766..4ef9902 100644
--- a/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java
+++ b/src/test/org/apache/hadoop/mapred/lib/TestCombineFileInputFormat.java
@@ -898,6 +898,28 @@ public class TestCombineFileInputFormat extends TestCase{
     }
   }
 
+  public void testGetConfiguredMaxSplitSize() throws Throwable {
+    JobConf conf = new JobConf();
+    DummyInputFormat inFormat = new DummyInputFormat();
+    // if both mapred.max.split.size and
+    // mapreduce.input.fileinputformat.split.maxsize are not set, return 0.
+    long maxSize = inFormat.getConfiguredMaxSplitSize(conf);
+    assertEquals(maxSize, 0L);
+
+    // if only mapreduce.input.fileinputformat.split.maxsize is set,
+    // return the value of mapreduce.input.fileinputformat.split.maxsize.
+    conf.setLong("mapreduce.input.fileinputformat.split.maxsize", 100L);
+    maxSize = inFormat.getConfiguredMaxSplitSize(conf);
+    assertEquals(maxSize, 100L);
+
+    // if both mapred.max.split.size and
+    // mapreduce.input.fileinputformat.split.maxsize are set,
+    // return the value of mapred.max.split.size.
+    conf.setLong("mapred.max.split.size", 1000L);
+    maxSize = inFormat.getConfiguredMaxSplitSize(conf);
+    assertEquals(maxSize, 1000L);
+  }
+
   static class TestFilter implements PathFilter {
     private Path p;
 


Mime
View raw message