Author: cdouglas Date: Mon Oct 19 00:19:23 2009 New Revision: 826566 URL: http://svn.apache.org/viewvc?rev=826566&view=rev Log: MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet. Contributed by Todd Lipcon Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=826566&r1=826565&r2=826566&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original) +++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Mon Oct 19 00:19:23 2009 @@ -767,3 +767,6 @@ MAPREDUCE-1041. Make TaskInProgress::taskStatuses map package-private. (Jothi Padmanabhan via cdouglas) + + MAPREDUCE-1070. Prevent a deadlock in the fair scheduler servlet. + (Todd Lipcon via cdouglas) Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=826566&r1=826565&r2=826566&view=diff ============================================================================== --- hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java (original) +++ hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java Mon Oct 19 00:19:23 2009 @@ -18,7 +18,9 @@ package org.apache.hadoop.mapred; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -118,7 +120,12 @@ } // Print out the normal response response.setContentType("text/html"); - PrintWriter out = new PrintWriter(response.getOutputStream()); + + // Because the client may read arbitrarily slow, and we hold locks while + // the servlet outputs, we want to write to our own buffer which we know + // won't block. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter out = new PrintWriter(baos); String hostname = StringUtils.simpleHostname( jobTracker.getJobTrackerMachine()); out.print(""); @@ -132,6 +139,11 @@ showJobs(out, advancedView); out.print("\n"); out.close(); + + // Flush our buffer to the real servlet output + OutputStream servletOut = response.getOutputStream(); + baos.writeTo(servletOut); + servletOut.close(); } /** @@ -202,55 +214,57 @@ out.print("FinishedRunningFair Share" + (advancedView ? "Weight" : "")); out.print("\n"); - Collection runningJobs = jobTracker.getRunningJobs(); - synchronized (scheduler) { - for (JobInProgress job: runningJobs) { - JobProfile profile = job.getProfile(); - JobInfo info = scheduler.infos.get(job); - if (info == null) { // Job finished, but let's show 0's for info - info = new JobInfo(null, null); - } - out.print("\n"); - out.printf("%s\n", DATE_FORMAT.format( - new Date(job.getStartTime()))); - out.printf("%s", - profile.getJobID(), profile.getJobID()); - out.printf("%s\n", profile.getUser()); - out.printf("%s\n", profile.getJobName()); - if (JSPUtil.privateActionsAllowed()) { - out.printf("%s\n", generateSelect(scheduler - .getPoolManager().getPoolNames(), scheduler.getPoolManager() - .getPoolName(job), "/scheduler?setPool=&jobid=" - + profile.getJobID() + (advancedView ? "&advanced" : ""))); - out.printf("%s\n", generateSelect(Arrays - .asList(new String[] { "VERY_LOW", "LOW", "NORMAL", "HIGH", - "VERY_HIGH" }), job.getPriority().toString(), - "/scheduler?setPriority=&jobid=" + profile.getJobID() - + (advancedView ? "&advanced" : ""))); - } else { - out.printf("%s\n", scheduler.getPoolManager().getPoolName(job)); - out.printf("%s\n", job.getPriority().toString()); - } - Pool pool = scheduler.getPoolManager().getPool(job); - String mapShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ? - String.format("%.1f", info.mapSchedulable.getFairShare()) : "NA"; - out.printf("%d / %d%d%s\n", - job.finishedMaps(), job.desiredMaps(), - info.mapSchedulable.getRunningTasks(), - mapShare); - if (advancedView) { - out.printf("%.1f\n", info.mapSchedulable.getWeight()); - } - String reduceShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ? - String.format("%.1f", info.reduceSchedulable.getFairShare()) : "NA"; - out.printf("%d / %d%d%s\n", - job.finishedReduces(), job.desiredReduces(), - info.reduceSchedulable.getRunningTasks(), - reduceShare); - if (advancedView) { - out.printf("%.1f\n", info.reduceSchedulable.getWeight()); + synchronized (jobTracker) { + Collection runningJobs = jobTracker.getRunningJobs(); + synchronized (scheduler) { + for (JobInProgress job: runningJobs) { + JobProfile profile = job.getProfile(); + JobInfo info = scheduler.infos.get(job); + if (info == null) { // Job finished, but let's show 0's for info + info = new JobInfo(null, null); + } + out.print("\n"); + out.printf("%s\n", DATE_FORMAT.format( + new Date(job.getStartTime()))); + out.printf("%s", + profile.getJobID(), profile.getJobID()); + out.printf("%s\n", profile.getUser()); + out.printf("%s\n", profile.getJobName()); + if (JSPUtil.privateActionsAllowed()) { + out.printf("%s\n", generateSelect(scheduler + .getPoolManager().getPoolNames(), scheduler.getPoolManager() + .getPoolName(job), "/scheduler?setPool=&jobid=" + + profile.getJobID() + (advancedView ? "&advanced" : ""))); + out.printf("%s\n", generateSelect(Arrays + .asList(new String[] { "VERY_LOW", "LOW", "NORMAL", "HIGH", + "VERY_HIGH" }), job.getPriority().toString(), + "/scheduler?setPriority=&jobid=" + profile.getJobID() + + (advancedView ? "&advanced" : ""))); + } else { + out.printf("%s\n", scheduler.getPoolManager().getPoolName(job)); + out.printf("%s\n", job.getPriority().toString()); + } + Pool pool = scheduler.getPoolManager().getPool(job); + String mapShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ? + String.format("%.1f", info.mapSchedulable.getFairShare()) : "NA"; + out.printf("%d / %d%d%s\n", + job.finishedMaps(), job.desiredMaps(), + info.mapSchedulable.getRunningTasks(), + mapShare); + if (advancedView) { + out.printf("%.1f\n", info.mapSchedulable.getWeight()); + } + String reduceShare = (pool.getSchedulingMode() == SchedulingMode.FAIR) ? + String.format("%.1f", info.reduceSchedulable.getFairShare()) : "NA"; + out.printf("%d / %d%d%s\n", + job.finishedReduces(), job.desiredReduces(), + info.reduceSchedulable.getRunningTasks(), + reduceShare); + if (advancedView) { + out.printf("%.1f\n", info.reduceSchedulable.getWeight()); + } + out.print("\n"); } - out.print("\n"); } } out.print("\n");