hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r826565 - in /hadoop/mapreduce/trunk: CHANGES.txt src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Date Mon, 19 Oct 2009 00:19:16 GMT
Author: cdouglas
Date: Mon Oct 19 00:19:15 2009
New Revision: 826565

URL: http://svn.apache.org/viewvc?rev=826565&view=rev
Log:
MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
Contributed by Todd Lipcon

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=826565&r1=826564&r2=826565&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Oct 19 00:19:15 2009
@@ -791,3 +791,6 @@
 
     MAPREDUCE-1041. Make TaskInProgress::taskStatuses map package-private.
     (Jothi Padmanabhan via cdouglas)
+
+    MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet.
+    (Todd Lipcon via cdouglas)

Modified: hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=826565&r1=826564&r2=826565&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Mon Oct 19 00:19:15 2009
@@ -18,7 +18,9 @@
 
 package org.apache.hadoop.mapred;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
@@ -118,7 +120,12 @@
     }
     // Print out the normal response
     response.setContentType("text/html");
-    PrintWriter out = new PrintWriter(response.getOutputStream());
+
+    // Because the client may read arbitrarily slow, and we hold locks while
+    // the servlet outputs, we want to write to our own buffer which we know
+    // won't block.
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter out = new PrintWriter(baos);
     String hostname = StringUtils.simpleHostname(
         jobTracker.getJobTrackerMachine());
     out.print("<html><head>");
@@ -132,6 +139,11 @@
     showJobs(out, advancedView);
     out.print("</body></html>\n");
     out.close();
+
+    // Flush our buffer to the real servlet output
+    OutputStream servletOut = response.getOutputStream();
+    baos.writeTo(servletOut);
+    servletOut.close();
   }
 
   /**
@@ -202,55 +214,57 @@
     out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>"
+
         (advancedView ? "<th>Weight</th>" : ""));
     out.print("</tr>\n");
-    Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
-    synchronized (scheduler) {
-      for (JobInProgress job: runningJobs) {
-        JobProfile profile = job.getProfile();
-        JobInfo info = scheduler.infos.get(job);
-        if (info == null) { // Job finished, but let's show 0's for info
-          info = new JobInfo(null, null);
-        }
-        out.print("<tr>\n");
-        out.printf("<td>%s</td>\n", DATE_FORMAT.format(
-            new Date(job.getStartTime())));
-        out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
-            profile.getJobID(), profile.getJobID());
-        out.printf("<td>%s</td>\n", profile.getUser());
-        out.printf("<td>%s</td>\n", profile.getJobName());
-        if (JSPUtil.privateActionsAllowed()) {
-          out.printf("<td>%s</td>\n", generateSelect(scheduler
-              .getPoolManager().getPoolNames(), scheduler.getPoolManager()
-              .getPoolName(job), "/scheduler?setPool=<CHOICE>&jobid="
-              + profile.getJobID() + (advancedView ? "&advanced" : "")));
-          out.printf("<td>%s</td>\n", generateSelect(Arrays
-              .asList(new String[] { "VERY_LOW", "LOW", "NORMAL", "HIGH",
-                  "VERY_HIGH" }), job.getPriority().toString(),
-              "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID()
-                  + (advancedView ? "&advanced" : "")));
-        } else {
-          out.printf("<td>%s</td>\n", scheduler.getPoolManager().getPoolName(job));
-          out.printf("<td>%s</td>\n", job.getPriority().toString());
-        }
-        Pool pool = scheduler.getPoolManager().getPool(job);
-        String mapShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ?
-            String.format("%.1f", info.mapSchedulable.getFairShare()) : "NA";
-        out.printf("<td>%d / %d</td><td>%d</td><td>%s</td>\n",
-            job.finishedMaps(), job.desiredMaps(), 
-            info.mapSchedulable.getRunningTasks(),
-            mapShare);
-        if (advancedView) {
-          out.printf("<td>%.1f</td>\n", info.mapSchedulable.getWeight());
-        }
-        String reduceShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ?
-            String.format("%.1f", info.reduceSchedulable.getFairShare()) : "NA";
-        out.printf("<td>%d / %d</td><td>%d</td><td>%s</td>\n",
-            job.finishedReduces(), job.desiredReduces(), 
-            info.reduceSchedulable.getRunningTasks(),
-            reduceShare);
-        if (advancedView) {
-          out.printf("<td>%.1f</td>\n", info.reduceSchedulable.getWeight());
+    synchronized (jobTracker) {
+      Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+      synchronized (scheduler) {
+        for (JobInProgress job: runningJobs) {
+          JobProfile profile = job.getProfile();
+          JobInfo info = scheduler.infos.get(job);
+          if (info == null) { // Job finished, but let's show 0's for info
+            info = new JobInfo(null, null);
+          }
+          out.print("<tr>\n");
+          out.printf("<td>%s</td>\n", DATE_FORMAT.format(
+              new Date(job.getStartTime())));
+          out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
+              profile.getJobID(), profile.getJobID());
+          out.printf("<td>%s</td>\n", profile.getUser());
+          out.printf("<td>%s</td>\n", profile.getJobName());
+          if (JSPUtil.privateActionsAllowed()) {
+            out.printf("<td>%s</td>\n", generateSelect(scheduler
+                .getPoolManager().getPoolNames(), scheduler.getPoolManager()
+                .getPoolName(job), "/scheduler?setPool=<CHOICE>&jobid="
+                + profile.getJobID() + (advancedView ? "&advanced" : "")));
+            out.printf("<td>%s</td>\n", generateSelect(Arrays
+                .asList(new String[] { "VERY_LOW", "LOW", "NORMAL", "HIGH",
+                    "VERY_HIGH" }), job.getPriority().toString(),
+                "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID()
+                    + (advancedView ? "&advanced" : "")));
+          } else {
+            out.printf("<td>%s</td>\n", scheduler.getPoolManager().getPoolName(job));
+            out.printf("<td>%s</td>\n", job.getPriority().toString());
+          }
+          Pool pool = scheduler.getPoolManager().getPool(job);
+          String mapShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ?
+              String.format("%.1f", info.mapSchedulable.getFairShare()) : "NA";
+          out.printf("<td>%d / %d</td><td>%d</td><td>%s</td>\n",
+              job.finishedMaps(), job.desiredMaps(), 
+              info.mapSchedulable.getRunningTasks(),
+              mapShare);
+          if (advancedView) {
+            out.printf("<td>%.1f</td>\n", info.mapSchedulable.getWeight());
+          }
+          String reduceShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ?
+              String.format("%.1f", info.reduceSchedulable.getFairShare()) : "NA";
+          out.printf("<td>%d / %d</td><td>%d</td><td>%s</td>\n",
+              job.finishedReduces(), job.desiredReduces(), 
+              info.reduceSchedulable.getRunningTasks(),
+              reduceShare);
+          if (advancedView) {
+            out.printf("<td>%.1f</td>\n", info.reduceSchedulable.getWeight());
+          }
+          out.print("</tr>\n");
         }
-        out.print("</tr>\n");
       }
     }
     out.print("</table>\n");



Mime
View raw message