hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r703294 - in /hadoop/core/branches/branch-0.19: ./ conf/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/
Date Thu, 09 Oct 2008 23:52:23 GMT
Author: dhruba
Date: Thu Oct  9 16:52:23 2008
New Revision: 703294

URL: http://svn.apache.org/viewvc?rev=703294&view=rev
Log:
HADOOP-4018. The number of tasks for a single job cannot exceed a 
pre-configured maximum value. (dhruba)


Added:
    hadoop/core/branches/branch-0.19/src/test/org/apache/hadoop/mapred/TestTaskLimits.java
      - copied unchanged from r703292, hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestTaskLimits.java
Modified:
    hadoop/core/branches/branch-0.19/CHANGES.txt
    hadoop/core/branches/branch-0.19/conf/hadoop-default.xml
    hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java

Modified: hadoop/core/branches/branch-0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/CHANGES.txt?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.19/CHANGES.txt Thu Oct  9 16:52:23 2008
@@ -819,6 +819,9 @@
     HADOOP-4267. Occasional exceptions during shutting down HSQLDB is logged 
     but not rethrown. (enis) 
 
+    HADOOP-4018. The number of tasks for a single job cannot exceed a 
+    pre-configured maximum value. (dhruba)
+
 Release 0.18.2 - Unreleased
 
   BUG FIXES

Modified: hadoop/core/branches/branch-0.19/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/conf/hadoop-default.xml?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/conf/hadoop-default.xml (original)
+++ hadoop/core/branches/branch-0.19/conf/hadoop-default.xml Thu Oct  9 16:52:23 2008
@@ -968,6 +968,13 @@
 </property>
 
 <property>
+  <name>mapred.jobtracker.maxtasks.per.job</name>
+  <value>-1</value>
+  <description>The maximum number of tasks for a single job.
+  A value of -1 indicates that there is no maximum.  </description>
+</property>
+
+<property>
   <name>mapred.submit.replication</name>
   <value>10</value>
   <description>The replication level for submitted job files.  This

Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Thu Oct  9 16:52:23 2008
@@ -52,7 +52,7 @@
  * ***********************************************************
  */
 class JobInProgress {
-  private static final Log LOG = LogFactory.getLog(JobInProgress.class);
+  static final Log LOG = LogFactory.getLog(JobInProgress.class);
     
   JobProfile profile;
   JobStatus status;
@@ -363,6 +363,18 @@
       splitFile.close();
     }
     numMapTasks = splits.length;
+
+
+    // if the number of splits is larger than a configured value
+    // then fail the job.
+    int maxTasks = jobtracker.getMaxTasksPerJob();
+    if (maxTasks > 0 && numMapTasks + numReduceTasks > maxTasks) {
+      throw new IOException(
+                "The number of tasks for this job " + 
+                (numMapTasks + numReduceTasks) +
+                " exceeds the configured limit " + maxTasks);
+    }
+
     maps = new TaskInProgress[numMapTasks];
     for(int i=0; i < numMapTasks; ++i) {
       inputLength += splits[i].getDataLength();

Modified: hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=703294&r1=703293&r2=703294&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/branches/branch-0.19/src/mapred/org/apache/hadoop/mapred/JobTracker.java Thu
Oct  9 16:52:23 2008
@@ -2721,5 +2721,12 @@
     return (JobStatus[]) jobStatusList.toArray(
         new JobStatus[jobStatusList.size()]);
   }
+
+  /**
+   * Returns the confgiured maximum number of tasks for a single job
+   */
+  int getMaxTasksPerJob() {
+    return conf.getInt("mapred.jobtracker.maxtasks.per.job", -1);
+  }
   
 }



Mime
View raw message