hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r532855 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/mapred/JobTracker.java
Date Thu, 26 Apr 2007 20:50:33 GMT
Author: cutting
Date: Thu Apr 26 13:50:32 2007
New Revision: 532855

URL: http://svn.apache.org/viewvc?view=rev&rev=532855
Log:
HADOOP-1279.  Fix JobTracker to maintain list of recently completed jobs by order of completion,
not submission.  Contributed by Arun.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=532855&r1=532854&r2=532855
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Apr 26 13:50:32 2007
@@ -259,6 +259,10 @@
     of the checksum file when a checksum error is detected.  
     (Hairong Kuang via tomwhite)
 
+78. HADOOP-1279.  Fix JobTracker to maintain list of recently
+    completed jobs by order of completion, not submission.
+    (Arun C Murthy via cutting)
+
 
 Release 0.12.3 - 2007-04-06
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?view=diff&rev=532855&r1=532854&r2=532855
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Apr 26 13:50:32
2007
@@ -353,15 +353,20 @@
                       jobs.remove(job.getProfile().getJobId());
                       jobInitQueue.remove(job);
                       jobsByArrival.remove(job);
+                      String jobUser = job.getProfile().getUser();
                       synchronized (userToJobsMap) {
                         ArrayList<JobInProgress> userJobs =
-                          userToJobsMap.get(job.getProfile().getUser());
+                          userToJobsMap.get(jobUser);
                         synchronized (userJobs) {
                           userJobs.remove(job);
                         }
+                        if (userJobs.isEmpty()) {
+                          userToJobsMap.remove(jobUser);
+                        }
                       }
                       LOG.info("Retired job with id: '" + 
-                               job.getProfile().getJobId() + "'");
+                               job.getProfile().getJobId() + "' of user '" +
+                               jobUser + "'");
                     }
                   }
                 }
@@ -895,11 +900,19 @@
     synchronized (jobs) {
       synchronized (jobsByArrival) {
         synchronized (jobInitQueue) {
-          String jobUser = job.getProfile().getUser();
           synchronized (userToJobsMap) {
+            String jobUser = job.getProfile().getUser();
+            if (!userToJobsMap.containsKey(jobUser)) {
+              userToJobsMap.put(jobUser, 
+                                new ArrayList<JobInProgress>());
+            }
             ArrayList<JobInProgress> userJobs = 
               userToJobsMap.get(jobUser);
             synchronized (userJobs) {
+              // Add the currently completed 'job'
+              userJobs.add(job);
+
+              // Check if we need to retire some jobs of this user
               while (userJobs.size() > 
                      MAX_COMPLETE_USER_JOBS_IN_MEMORY) {
                 JobInProgress rjob = userJobs.get(0);
@@ -929,7 +942,8 @@
                   jobsByArrival.remove(rjob);
                     
                   LOG.info("Retired job with id: '" + 
-                           rjob.getProfile().getJobId() + "'");
+                           rjob.getProfile().getJobId() + "' of user: '" +
+                           jobUser + "'");
                 } else {
                   // Do not remove jobs that aren't complete.
                   // Stop here, and let the next pass take
@@ -938,6 +952,9 @@
                 }
               }
             }
+            if (userJobs.isEmpty()) {
+              userToJobsMap.remove(jobUser);
+            }
           }
         }
       }
@@ -1424,22 +1441,10 @@
     synchronized (jobs) {
       synchronized (jobsByArrival) {
         synchronized (jobInitQueue) {
-          synchronized (userToJobsMap) {
-            jobs.put(job.getProfile().getJobId(), job);
-            String jobUser = job.getProfile().getUser();
-            if (!userToJobsMap.containsKey(jobUser)) {
-              userToJobsMap.put(jobUser, 
-                                new ArrayList<JobInProgress>());
-            }
-            ArrayList<JobInProgress> userJobs = 
-              userToJobsMap.get(jobUser);
-            synchronized (userJobs) {
-              userJobs.add(job);
-            }
-            jobsByArrival.add(job);
-            jobInitQueue.add(job);
-            jobInitQueue.notifyAll();
-          }
+          jobs.put(job.getProfile().getJobId(), job);
+          jobsByArrival.add(job);
+          jobInitQueue.add(job);
+          jobInitQueue.notifyAll();
         }
       }
     }



Mime
View raw message