hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r1067363 - in /hadoop/mapreduce/branches/branch-0.22: ./ src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapred/
Date Sat, 05 Feb 2011 00:44:40 GMT
Author: shv
Date: Sat Feb  5 00:44:40 2011
New Revision: 1067363

URL: http://svn.apache.org/viewvc?rev=1067363&view=rev
Log:
MAPREDUCE-1825. Merge r1067361 from trunk to branch 0.22.

Modified:
    hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
    hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
    hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java
    hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java

Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Sat Feb  5 00:44:40 2011
@@ -475,6 +475,10 @@ Release 0.22.0 - Unreleased
     MAPREDUCE-2304. Fix TestMRCLI to allow hostname with a hyphen (-).
     (Priyo Mustafi via shv)
 
+    MAPREDUCE-1825. jobqueue_details.jsp and FairSchedulerServelet should not
+    call finishedMaps and finishedReduces when job is not initialized.
+    (Scott Chen via shv)
+
 Release 0.21.1 - Unreleased
 
   NEW FEATURES

Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
Sat Feb  5 00:44:40 2011
@@ -676,6 +676,9 @@ public class FairScheduler extends TaskS
     Map<String, Integer> userJobs = new HashMap<String, Integer>();
     Map<String, Integer> poolJobs = new HashMap<String, Integer>();
     for (JobInProgress job: jobs) {
+      if (!job.inited()) {
+        continue;
+      }
       String user = job.getJobConf().getUser();
       String pool = poolMgr.getPoolName(job);
       int userCount = userJobs.containsKey(user) ? userJobs.get(user) : 0;

Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Sat Feb  5 00:44:40 2011
@@ -30,6 +30,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.List;
 
 import javax.servlet.ServletContext;
@@ -83,7 +84,7 @@ public class FairSchedulerServlet extend
     boolean advancedView = request.getParameter("advanced") != null;
     if (JSPUtil.privateActionsAllowed(jobTracker.conf)
         && request.getParameter("setPool") != null) {
-      Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+      Collection<JobInProgress> runningJobs = getInitedJobs();
       PoolManager poolMgr = null;
       synchronized (scheduler) {
         poolMgr = scheduler.getPoolManager();
@@ -104,7 +105,7 @@ public class FairSchedulerServlet extend
     }
     if (JSPUtil.privateActionsAllowed(jobTracker.conf)
         && request.getParameter("setPriority") != null) {
-      Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();      
+      Collection<JobInProgress> runningJobs = getInitedJobs();
       JobPriority priority = JobPriority.valueOf(request.getParameter(
           "setPriority"));
       String jobId = request.getParameter("jobid");
@@ -246,7 +247,7 @@ public class FairSchedulerServlet extend
         (advancedView ? "<th>Weight</th>" : ""));
     out.print("</tr>\n");
     synchronized (jobTracker) {
-      Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+      Collection<JobInProgress> runningJobs = getInitedJobs();
       synchronized (scheduler) {
         for (JobInProgress job: runningJobs) {
           JobProfile profile = job.getProfile();
@@ -323,4 +324,19 @@ public class FairSchedulerServlet extend
     html.append("</select>\n");
     return html.toString();
   }
+
+  /**
+   * Obtained all initialized jobs
+   */
+  private Collection<JobInProgress> getInitedJobs() {
+    Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+    for (Iterator<JobInProgress> it = runningJobs.iterator(); it.hasNext();) {
+      JobInProgress job = it.next();
+      if (!job.inited()) {
+        it.remove();
+      }
+    }
+    return runningJobs;
+  }
+
 }

Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/JobSchedulable.java
Sat Feb  5 00:44:40 2011
@@ -99,6 +99,9 @@ public class JobSchedulable extends Sche
 
   @Override
   public int getRunningTasks() {
+    if (!job.inited()) {
+      return 0;
+    }
     return taskType == TaskType.MAP ? job.runningMaps() : job.runningReduces();
   }
 

Modified: hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java?rev=1067363&r1=1067362&r2=1067363&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java (original)
+++ hadoop/mapreduce/branches/branch-0.22/src/java/org/apache/hadoop/mapred/JSPUtil.java Sat
Feb  5 00:44:40 2011
@@ -264,6 +264,15 @@ class JSPUtil {
   public static String generateJobTable(String label, Collection<JobInProgress> jobs
       , int refresh, int rowId, JobConf conf) throws IOException {
 
+    // Remove uninitialized jobs because calling JobInProgress synchronized
+    // methods while job initialization takes long time
+    for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext();) {
+      JobInProgress job = it.next();
+      if (!job.inited()) {
+        it.remove();
+      }
+    }
+
     boolean isModifiable = label.equals("Running") &&
         privateActionsAllowed(conf);
     StringBuilder sb = new StringBuilder();



Mime
View raw message