hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject [30/50] [abbrv] hadoop git commit: MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value greater than 2GB expressed in bytes. (Hao Xia via kasha)
Date Fri, 01 May 2015 07:01:47 GMT
MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value greater than 2GB expressed
in bytes. (Hao Xia via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51909232
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51909232
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51909232

Branch: refs/heads/HDFS-7240
Commit: 519092322dd1bf71984bef1393d8e082643408cd
Parents: bc1bd7e
Author: Karthik Kambatla <kasha@apache.org>
Authored: Tue Apr 28 14:05:26 2015 -0700
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Tue Apr 28 14:05:26 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt                |  3 +++
 .../main/java/org/apache/hadoop/mapred/JobConf.java | 10 +++++-----
 .../java/org/apache/hadoop/mapred/TestJobConf.java  | 16 ++++++++++++++++
 3 files changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/51909232/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 2090007..e9e1a4b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -188,6 +188,9 @@ Trunk (Unreleased)
     MAPREDUCE-6234. TestHighRamJob fails due to the change in MAPREDUCE-5785. 
     (Masatake Iwasaki via kasha)
 
+    MAPREDUCE-6343. JobConf.parseMaximumHeapSizeMB() fails to parse value 
+    greater than 2GB expressed in bytes. (Hao Xia via kasha)
+
   BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
 
     MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by

http://git-wip-us.apache.org/repos/asf/hadoop/blob/51909232/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index 9cac685..059593a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -2080,28 +2080,28 @@ public class JobConf extends Configuration {
     // Find the last matching -Xmx following word boundaries
     Matcher m = JAVA_OPTS_XMX_PATTERN.matcher(javaOpts);
     if (m.matches()) {
-      int size = Integer.parseInt(m.group(1));
+      long size = Long.parseLong(m.group(1));
       if (size <= 0) {
         return -1;
       }
       if (m.group(2).isEmpty()) {
         // -Xmx specified in bytes
-        return size / (1024 * 1024);
+        return (int) (size / (1024 * 1024));
       }
       char unit = m.group(2).charAt(0);
       switch (unit) {
         case 'g':
         case 'G':
           // -Xmx specified in GB
-          return size * 1024;
+          return (int) (size * 1024);
         case 'm':
         case 'M':
           // -Xmx specified in MB
-          return size;
+          return (int) size;
         case 'k':
         case 'K':
           // -Xmx specified in KB
-          return size / 1024;
+          return (int) (size / 1024);
       }
     }
     // -Xmx not specified

http://git-wip-us.apache.org/repos/asf/hadoop/blob/51909232/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
index 0612ade..57fa4e6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
@@ -361,4 +361,20 @@ public class TestJobConf {
       jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
       );
   }
+
+  /**
+   * Test parsing various types of Java heap options.
+   */
+  @Test
+  public void testParseMaximumHeapSizeMB() {
+    // happy cases
+    Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4294967296"));
+    Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4194304k"));
+    Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4096m"));
+    Assert.assertEquals(4096, JobConf.parseMaximumHeapSizeMB("-Xmx4g"));
+
+    // sad cases
+    Assert.assertEquals(-1, JobConf.parseMaximumHeapSizeMB("-Xmx4?"));
+    Assert.assertEquals(-1, JobConf.parseMaximumHeapSizeMB(""));
+  }
 }


Mime
View raw message