hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yhema...@apache.org
Subject svn commit: r747805 - in /hadoop/core/branches/branch-0.20: ./ CHANGES.txt src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Date Wed, 25 Feb 2009 14:59:24 GMT
Author: yhemanth
Date: Wed Feb 25 14:59:24 2009
New Revision: 747805

URL: http://svn.apache.org/viewvc?rev=747805&view=rev
Log:
Merge -r 747801:747802 from trunk to branch 0.20 to fix HADOOP-5154.

Modified:
    hadoop/core/branches/branch-0.20/   (props changed)
    hadoop/core/branches/branch-0.20/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
    hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java

Propchange: hadoop/core/branches/branch-0.20/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 25 14:59:24 2009
@@ -1,2 +1,2 @@
 /hadoop/core/branches/branch-0.19:713112
-/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279
+/hadoop/core/trunk:727001,727117,727191,727212,727217,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,736426,738328,738697,740077,740157,741703,741762,743745,743816,743892,744894,745180,746010,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279,747802

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=747805&r1=747804&r2=747805&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Wed Feb 25 14:59:24 2009
@@ -661,6 +661,13 @@
     HADOOP-5272. Fixes a problem to do with detecting whether an attempt is the first
     attempt of a Task. This affects JobTracker restart. (Amar Kamat via ddas)
 
+Release 0.19.2 - Unreleased
+
+  BUG FIXES
+
+    HADOOP-5154. Fixes a deadlock in the fairshare scheduler. 
+    (Matei Zaharia via yhemanth)
+    
 Release 0.19.1 - Unreleased
 
   IMPROVEMENTS

Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 25 14:59:24 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732613,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296,743745,743816,743892,744894,745180,745268,746010,746193,746206,746227,746233,746274,746902-746903,746944,746968,746970,747279,747802

Modified: hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java?rev=747805&r1=747804&r2=747805&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
(original)
+++ hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairScheduler.java
Wed Feb 25 14:59:24 2009
@@ -708,11 +708,11 @@
       clusterStatus.getMaxMapTasks() : clusterStatus.getMaxReduceTasks());
   }
 
-  public boolean getUseFifo() {
+  public synchronized boolean getUseFifo() {
     return useFifo;
   }
   
-  public void setUseFifo(boolean useFifo) {
+  public synchronized void setUseFifo(boolean useFifo) {
     this.useFifo = useFifo;
   }
   

Modified: hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java?rev=747805&r1=747804&r2=747805&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
(original)
+++ hadoop/core/branches/branch-0.20/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/FairSchedulerServlet.java
Wed Feb 25 14:59:24 2009
@@ -86,11 +86,11 @@
       return;
     }
     if (request.getParameter("setPool") != null) {
-      PoolManager poolMgr = scheduler.getPoolManager();
-      synchronized (poolMgr) {
+      Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+      synchronized (scheduler) {
+        PoolManager poolMgr = scheduler.getPoolManager();
         String pool = request.getParameter("setPool");
         String jobId = request.getParameter("jobid");
-        Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
         for (JobInProgress job: runningJobs) {
           if (job.getProfile().getJobID().toString().equals(jobId)) {
             poolMgr.setPool(job, pool);
@@ -103,12 +103,11 @@
       return;
     }
     if (request.getParameter("setPriority") != null) {
-      PoolManager poolMgr = scheduler.getPoolManager();
-      synchronized (poolMgr) {
+      Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+      synchronized (scheduler) {
         JobPriority priority = JobPriority.valueOf(request.getParameter(
             "setPriority"));
         String jobId = request.getParameter("jobid");
-        Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
         for (JobInProgress job: runningJobs) {
           if (job.getProfile().getJobID().toString().equals(jobId)) {
             job.setPriority(priority);
@@ -127,8 +126,6 @@
         jobTracker.getJobTrackerMachine());
     out.print("<html><head>");
     out.printf("<title>%s Job Scheduler Admininstration</title>\n", hostname);
-    out.printf("<META http-equiv=\"refresh\" content=\"15;URL=/scheduler%s\">",
-        advancedView ? "?advanced" : "");
     out.print("<link rel=\"stylesheet\" type=\"text/css\" " + 
         "href=\"/static/hadoop.css\">\n");
     out.print("</head><body>\n");
@@ -145,8 +142,8 @@
    * Print a view of pools to the given output writer.
    */
   private void showPools(PrintWriter out, boolean advancedView) {
-    PoolManager poolManager = scheduler.getPoolManager();
-    synchronized(poolManager) {
+    synchronized(scheduler) {
+      PoolManager poolManager = scheduler.getPoolManager();
       out.print("<h2>Pools</h2>\n");
       out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
       out.print("<tr><th>Pool</th><th>Running Jobs</th>" +

@@ -208,50 +205,52 @@
     out.print("<th>Finished</th><th>Running</th><th>Fair Share</th>"
+
         (advancedView ? "<th>Weight</th><th>Deficit</th><th>minReduces</th>"
: ""));
     out.print("</tr>\n");
-    Collection<JobInProgress> runningJobs = jobTracker.runningJobs();
-    for (JobInProgress job: runningJobs) {
-      JobProfile profile = job.getProfile();
-      JobInfo info = scheduler.infos.get(job);
-      if (info == null) { // Job finished, but let's show 0's for info
-        info = new JobInfo();
-      }
-      out.print("<tr>\n");
-      out.printf("<td>%s</td>\n", DATE_FORMAT.format(
-          new Date(job.getStartTime())));
-      out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
-          profile.getJobID(), profile.getJobID());
-      out.printf("<td>%s</td>\n", profile.getUser());
-      out.printf("<td>%s</td>\n", profile.getJobName());
-      out.printf("<td>%s</td>\n", generateSelect(
-          scheduler.getPoolManager().getPoolNames(),
-          scheduler.getPoolManager().getPoolName(job),
-          "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
-          (advancedView ? "&advanced" : "")));
-      out.printf("<td>%s</td>\n", generateSelect(
-          Arrays.asList(new String[]
-              {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
-          job.getPriority().toString(),
-          "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
-          (advancedView ? "&advanced" : "")));
-      out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
-          job.finishedMaps(), job.desiredMaps(), info.runningMaps,
-          info.mapFairShare);
-      if (advancedView) {
-        out.printf("<td>%8.1f</td>\n", info.mapWeight);
-        out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
-            (info.mapDeficit / 1000) + "s" : "--");
-        out.printf("<td>%d</td>\n", info.minMaps);
-      }
-      out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
-          job.finishedReduces(), job.desiredReduces(), info.runningReduces,
-          info.reduceFairShare);
-      if (advancedView) {
-        out.printf("<td>%8.1f</td>\n", info.reduceWeight);
-        out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
-            (info.reduceDeficit / 1000) + "s" : "--");
-        out.printf("<td>%d</td>\n", info.minReduces);
+    Collection<JobInProgress> runningJobs = jobTracker.getRunningJobs();
+    synchronized (scheduler) {
+      for (JobInProgress job: runningJobs) {
+        JobProfile profile = job.getProfile();
+        JobInfo info = scheduler.infos.get(job);
+        if (info == null) { // Job finished, but let's show 0's for info
+          info = new JobInfo();
+        }
+        out.print("<tr>\n");
+        out.printf("<td>%s</td>\n", DATE_FORMAT.format(
+            new Date(job.getStartTime())));
+        out.printf("<td><a href=\"jobdetails.jsp?jobid=%s\">%s</a></td>",
+            profile.getJobID(), profile.getJobID());
+        out.printf("<td>%s</td>\n", profile.getUser());
+        out.printf("<td>%s</td>\n", profile.getJobName());
+        out.printf("<td>%s</td>\n", generateSelect(
+            scheduler.getPoolManager().getPoolNames(),
+            scheduler.getPoolManager().getPoolName(job),
+            "/scheduler?setPool=<CHOICE>&jobid=" + profile.getJobID() +
+            (advancedView ? "&advanced" : "")));
+        out.printf("<td>%s</td>\n", generateSelect(
+            Arrays.asList(new String[]
+                {"VERY_LOW", "LOW", "NORMAL", "HIGH", "VERY_HIGH"}),
+            job.getPriority().toString(),
+            "/scheduler?setPriority=<CHOICE>&jobid=" + profile.getJobID() +
+            (advancedView ? "&advanced" : "")));
+        out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+            job.finishedMaps(), job.desiredMaps(), info.runningMaps,
+            info.mapFairShare);
+        if (advancedView) {
+          out.printf("<td>%8.1f</td>\n", info.mapWeight);
+          out.printf("<td>%s</td>\n", info.neededMaps > 0 ?
+              (info.mapDeficit / 1000) + "s" : "--");
+          out.printf("<td>%d</td>\n", info.minMaps);
+        }
+        out.printf("<td>%d / %d</td><td>%d</td><td>%8.1f</td>\n",
+            job.finishedReduces(), job.desiredReduces(), info.runningReduces,
+            info.reduceFairShare);
+        if (advancedView) {
+          out.printf("<td>%8.1f</td>\n", info.reduceWeight);
+          out.printf("<td>%s</td>\n", info.neededReduces > 0 ?
+              (info.reduceDeficit / 1000) + "s" : "--");
+          out.printf("<td>%d</td>\n", info.minReduces);
+        }
+        out.print("</tr>\n");
       }
-      out.print("</tr>\n");
     }
     out.print("</table>\n");
   }



Mime
View raw message