hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yhema...@apache.org
Subject svn commit: r937201 - in /hadoop/mapreduce/trunk: ./ src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapred/ src/test/
Date Fri, 23 Apr 2010 09:10:42 GMT
Author: yhemanth
Date: Fri Apr 23 09:10:42 2010
New Revision: 937201

URL: http://svn.apache.org/viewvc?rev=937201&view=rev
Log:
MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev targets and also fix
existing warnings. Contributed by Hong Tang.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/build.xml
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
    hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java
    hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Apr 23 09:10:42 2010
@@ -553,6 +553,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1409. IOExceptions thrown from FIleOutputCommitter::abortTask
     should cause the task to fail. (Amareshwari Sriramadasu via cdouglas)
 
+    MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev
+    targets and also fix existing warnings. (Hong Tang via yhemanth)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/build.xml?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/build.xml (original)
+++ hadoop/mapreduce/trunk/build.xml Fri Apr 23 09:10:42 2010
@@ -742,6 +742,7 @@
    <subant target="jar">
       <property name="version" value="${version}"/>
       <property name="dist.dir" value="${dist.dir}"/>
+      <fileset file="${contrib.dir}/capacity-scheduler/build.xml"/>
       <fileset file="${contrib.dir}/streaming/build.xml"/>
       <fileset file="${contrib.dir}/gridmix/build.xml"/>
       <fileset file="${contrib.dir}/mumak/build.xml"/>
@@ -773,12 +774,14 @@
       <sourcePath path="${mapred.src.dir}"/>
       <sourcePath path="${examples.dir}" />
       <sourcePath path="${tools.src}" />
+      <sourcePath path="${basedir}/src/contrib/capacity-scheduler/src/java" />
       <sourcePath path="${basedir}/src/contrib/streaming/src/java" />
       <sourcePath path="${basedir}/src/contrib/gridmix/src/java" />
       <sourcePath path="${basedir}/src/contrib/mumak/src/java" />
       <class location="${basedir}/build/${final.name}.jar" />
       <class location="${basedir}/build/${examples.final.name}.jar" />
       <class location="${basedir}/build/${tools.final.name}.jar" />
+      <class location="${basedir}/build/contrib/capacity-scheduler/hadoop-${version}-capacity-scheduler.jar"
/>
       <class location="${basedir}/build/contrib/streaming/hadoop-${version}-streaming.jar"
/>
       <class location="${basedir}/build/contrib/gridmix/hadoop-${version}-gridmix.jar"
/>
       <class location="${basedir}/build/contrib/mumak/hadoop-${version}-mumak.jar" />
@@ -836,6 +839,7 @@
         <packageset dir="${mapred.src.dir}"/>
     	<packageset dir="${examples.dir}"/>
 
+    	<packageset dir="src/contrib/capacity-scheduler/src/java"/>
     	<packageset dir="src/contrib/streaming/src/java"/>
     	<packageset dir="src/contrib/data_join/src/java"/>
     	<packageset dir="src/contrib/index/src/java"/>

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
Fri Apr 23 09:10:42 2010
@@ -578,16 +578,20 @@ class CapacityTaskScheduler extends Task
     @Override
     Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job)
     throws IOException {
-      ClusterStatus clusterStatus =
-        scheduler.taskTrackerManager.getClusterStatus();
-      int numTaskTrackers = clusterStatus.getTaskTrackers();
-      return job.obtainNewMapTask(taskTracker, numTaskTrackers,
-          scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+      synchronized (scheduler) {
+        ClusterStatus clusterStatus = scheduler.taskTrackerManager
+            .getClusterStatus();
+        int numTaskTrackers = clusterStatus.getTaskTrackers();
+        return job.obtainNewMapTask(taskTracker, numTaskTrackers,
+            scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+      }
     }
 
     @Override
     int getClusterCapacity() {
-      return scheduler.taskTrackerManager.getClusterStatus().getMaxMapTasks();
+      synchronized (scheduler) {
+        return scheduler.taskTrackerManager.getClusterStatus().getMaxMapTasks();
+      }
     }
 
     @Override
@@ -621,17 +625,21 @@ class CapacityTaskScheduler extends Task
     @Override
     Task obtainNewTask(TaskTrackerStatus taskTracker, JobInProgress job)
     throws IOException {
-      ClusterStatus clusterStatus =
-        scheduler.taskTrackerManager.getClusterStatus();
-      int numTaskTrackers = clusterStatus.getTaskTrackers();
-      return job.obtainNewReduceTask(taskTracker, numTaskTrackers,
-          scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+      synchronized (scheduler) {
+        ClusterStatus clusterStatus = scheduler.taskTrackerManager
+            .getClusterStatus();
+        int numTaskTrackers = clusterStatus.getTaskTrackers();
+        return job.obtainNewReduceTask(taskTracker, numTaskTrackers,
+            scheduler.taskTrackerManager.getNumberOfUniqueHosts());
+      }
     }
 
     @Override
     int getClusterCapacity() {
-      return scheduler.taskTrackerManager.getClusterStatus()
-          .getMaxReduceTasks();
+      synchronized (scheduler) {
+        return scheduler.taskTrackerManager.getClusterStatus()
+            .getMaxReduceTasks();
+      }
     }
 
     @Override
@@ -863,7 +871,7 @@ class CapacityTaskScheduler extends Task
   }
 
   /** mostly for testing purposes */
-  void setInitializationPoller(JobInitializationPoller p) {
+  synchronized void setInitializationPoller(JobInitializationPoller p) {
     this.initializationPoller = p;
   }
   
@@ -888,7 +896,7 @@ class CapacityTaskScheduler extends Task
    * provided for the test classes
    * lets you update the QSI objects and sorted collections
    */ 
-  void updateContextInfoForTests() {
+  synchronized void updateContextInfoForTests() {
     ClusterStatus c = taskTrackerManager.getClusterStatus();
     int mapClusterCapacity = c.getMaxMapTasks();
     int reduceClusterCapacity = c.getMaxReduceTasks();
@@ -1011,7 +1019,7 @@ class CapacityTaskScheduler extends Task
     return jobCollection;
   }
   
-  JobInitializationPoller getInitializationPoller() {
+  synchronized JobInitializationPoller getInitializationPoller() {
     return initializationPoller;
   }
 

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobInitializationPoller.java
Fri Apr 23 09:10:42 2010
@@ -191,6 +191,7 @@ public class JobInitializationPoller ext
       if (jobs == null) {
         LOG.error("Invalid queue passed to the thread : " + queue
             + " For job :: " + job.getJobID());
+        return;
       }
       synchronized (jobs) {
         JobSchedulingInfo schedInfo = new JobSchedulingInfo(job);
@@ -212,14 +213,12 @@ public class JobInitializationPoller ext
    * jobs allowed to be initialize per user in the queue.
    * 
    */
-  private class QueueInfo {
-    String queue;
+  private static class QueueInfo {
     int maxUsersAllowedToInitialize;
     int maxJobsPerUserToInitialize;
 
-    public QueueInfo(String queue, int maxUsersAllowedToInitialize,
+    public QueueInfo(int maxUsersAllowedToInitialize,
         int maxJobsPerUserToInitialize) {
-      this.queue = queue;
       this.maxJobsPerUserToInitialize = maxJobsPerUserToInitialize;
       this.maxUsersAllowedToInitialize = maxUsersAllowedToInitialize;
     }
@@ -290,8 +289,7 @@ public class JobInitializationPoller ext
       int maxJobsPerUserToInitialize =
           schedulerConf.getMaxJobsPerUserToInitialize(queue);
       QueueInfo qi =
-          new QueueInfo(queue, maxUsersToInitialize,
-              maxJobsPerUserToInitialize);
+          new QueueInfo(maxUsersToInitialize, maxJobsPerUserToInitialize);
       jobQueues.put(queue, qi);
     }
     sleepInterval = schedulerConf.getSleepInterval();

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/JobQueue.java
Fri Apr 23 09:10:42 2010
@@ -161,18 +161,17 @@ class JobQueue extends AbstractQueue {
       j.getProfile().getUser(),
       i.intValue() + numReduceSlotsForThisJob);
     if (LOG.isDebugEnabled()) {
-      LOG.debug(
-        String.format(
-          "updateQSI: job %s: run(m)=%d, "
+      synchronized (j) {
+        LOG.debug(String.format("updateQSI: job %s: run(m)=%d, "
             + "occupied(m)=%d, run(r)=%d, occupied(r)=%d, finished(m)=%d,"
             + " finished(r)=%d, failed(m)=%d, failed(r)=%d, "
-            + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j
-            .getJobID().toString(), numMapsRunningForThisJob,
-          numMapSlotsForThisJob, numReducesRunningForThisJob,
-          numReduceSlotsForThisJob, j
-            .finishedMaps(), j.finishedReduces(), j.failedMapTasks,
-          j.failedReduceTasks, j.speculativeMapTasks, j.speculativeReduceTasks,
-          j.numMapTasks, j.numReduceTasks));
+            + "spec(m)=%d, spec(r)=%d, total(m)=%d, total(r)=%d", j.getJobID()
+            .toString(), numMapsRunningForThisJob, numMapSlotsForThisJob,
+            numReducesRunningForThisJob, numReduceSlotsForThisJob, j
+                .finishedMaps(), j.finishedReduces(), j.failedMapTasks,
+            j.failedReduceTasks, j.speculativeMapTasks,
+            j.speculativeReduceTasks, j.numMapTasks, j.numReduceTasks));
+      }
     }
 
     /*
@@ -355,8 +354,6 @@ class JobQueue extends AbstractQueue {
   }
 
   public void jobUpdated(JobChangeEvent event) {
-    JobInProgress job = event.getJobInProgress();
-
     // Check if this is the status change
     if (event instanceof JobStatusChangeEvent) {
       jobStateChanged((JobStatusChangeEvent) event);

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/QueueSchedulingContext.java
Fri Apr 23 09:10:42 2010
@@ -101,12 +101,12 @@ public class QueueSchedulingContext {
   QueueSchedulingContext(
     String queueName, float capacityPercent, float maxCapacityPercent,
     int ulMin) {
-    this.setQueueName(new String(queueName));
+    this.setQueueName(queueName);
     this.setCapacityPercent(capacityPercent);
     this.setMaxCapacityPercent(maxCapacityPercent);
     this.setUlMin(ulMin);
-    this.setMapTSC(new TaskSchedulingContext(TaskType.MAP));
-    this.setReduceTSC(new TaskSchedulingContext(TaskType.REDUCE));
+    this.setMapTSC(new TaskSchedulingContext());
+    this.setReduceTSC(new TaskSchedulingContext());
   }
 
   /**

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskDataView.java
Fri Apr 23 09:10:42 2010
@@ -29,8 +29,6 @@ import org.apache.hadoop.mapreduce.TaskT
  *
  */
 abstract class TaskDataView {
-  TaskType type;
-
   abstract int getRunningTasks(JobInProgress job);
 
   abstract int getPendingTasks(JobInProgress job);
@@ -80,10 +78,7 @@ abstract class TaskDataView {
    * The data view for map tasks
    */
   static class MapTaskDataView extends TaskDataView {
-    TaskType type;
-
     MapTaskDataView() {
-      type = TaskType.MAP;
     }
 
     @Override
@@ -118,10 +113,7 @@ abstract class TaskDataView {
    *  The data view for reduce tasks
    */
   static class ReduceTaskDataView extends TaskDataView {
-    TaskType type;
-
     ReduceTaskDataView() {
-      type = TaskType.REDUCE;
     }
 
     @Override

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/TaskSchedulingContext.java
Fri Apr 23 09:10:42 2010
@@ -19,6 +19,7 @@ package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.mapreduce.TaskType;
 
+import java.util.Iterator;
 import java.util.Map;
 import java.util.HashMap;
 import java.util.Set;
@@ -40,8 +41,6 @@ import java.util.Set;
  * ********************************************************************
  */
 public class TaskSchedulingContext {
-
-  private TaskType type;
   /**
    * the actual capacity, which depends on how many slots are available
    * in the cluster at any given time.
@@ -67,10 +66,6 @@ public class TaskSchedulingContext {
       " %s running reduce tasks using %d reduce slots." +
       " %d additional slots reserved.";
 
-  public TaskSchedulingContext(TaskType type) {
-    this.type = type;
-  }
-
   /**
    * reset the variables associated with tasks
    */
@@ -188,11 +183,12 @@ public class TaskSchedulingContext {
   }
 
   private void updateNoOfSlotsOccupiedByUser(Map<String, Integer> nou) {
-    Set<String> keys = nou.keySet();
-    for (String key : keys) {
-      if (this.numSlotsOccupiedByUser.containsKey(key)) {
-        int currentVal = this.numSlotsOccupiedByUser.get(key);
-        this.numSlotsOccupiedByUser.put(key, currentVal + nou.get(key));
+    for (Iterator<Map.Entry<String, Integer>> it = nou.entrySet().iterator();
it.hasNext(); ) {
+      Map.Entry<String, Integer> entry = it.next();
+      String key = entry.getKey();
+      Integer currentVal = numSlotsOccupiedByUser.get(key);
+      if (currentVal != null) {
+        this.numSlotsOccupiedByUser.put(key, currentVal + entry.getValue());
       }
     }
   }

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
Fri Apr 23 09:10:42 2010
@@ -339,7 +339,7 @@ public class CapacityTestUtils {
     }
 
     @Override
-    public Task obtainNewMapTask(
+    public synchronized Task obtainNewMapTask(
       final TaskTrackerStatus tts, int clusterSize,
       int ignored) throws IOException {
       boolean areAllMapsRunning = (mapTaskCtr == numMapTasks);
@@ -385,7 +385,7 @@ public class CapacityTestUtils {
     }
 
     @Override
-    public Task obtainNewReduceTask(
+    public synchronized Task obtainNewReduceTask(
       final TaskTrackerStatus tts,
       int clusterSize, int ignored) throws IOException {
       boolean areAllReducesRunning = (redTaskCtr == numReduceTasks);

Modified: hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestRefreshOfQueues.java
Fri Apr 23 09:10:42 2010
@@ -91,6 +91,18 @@ public class TestRefreshOfQueues {
   }
 
   /**
+   * Helper method that ensures TaskScheduler is locked before calling
+   * {@link QueueManager#refreshQueues(Configuration, 
+   *    org.apache.hadoop.mapred.TaskScheduler.QueueRefresher)}.
+   */
+  private static void refreshQueues(QueueManager qm, Configuration conf,
+      TaskScheduler ts) throws IOException {
+    synchronized (ts) {
+      qm.refreshQueues(conf, ts.getQueueRefresher());
+    }
+  }
+
+  /**
    * @throws Throwable
    */
   @Test
@@ -160,8 +172,7 @@ public class TestRefreshOfQueues {
         queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
 
     // Now do scheduler refresh.
-    taskTrackerManager.getQueueManager().refreshQueues(null,
-        scheduler.getQueueRefresher());
+    refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
 
     allQueues = getAllQueues(scheduler);
 
@@ -251,9 +262,7 @@ public class TestRefreshOfQueues {
     QueueManagerTestUtils.writeQueueConfigurationFile(
       queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
 
-    taskTrackerManager.getQueueManager().refreshQueues(
-      null,
-      scheduler.getQueueRefresher());
+    refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
 
     job1 =
       taskTrackerManager.submitJobAndInit(
@@ -350,8 +359,7 @@ public class TestRefreshOfQueues {
         queueConfigFile.getAbsolutePath(), new JobQueueInfo[] { queues[0] });
 
     try {
-      taskTrackerManager.getQueueManager().refreshQueues(null,
-          scheduler.getQueueRefresher());
+      refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
     } catch (IOException ioe) {
       assertTrue(ioe.getMessage().contains(
           String.format(QueueHierarchyBuilder.TOTAL_CAPACITY_OVERFLOWN_MSG,
@@ -422,9 +430,7 @@ public class TestRefreshOfQueues {
     QueueManagerTestUtils.writeQueueConfigurationFile(
       queueConfigFile.getAbsolutePath(), new JobQueueInfo[]{queues[0]});
 
-    taskTrackerManager.getQueueManager().refreshQueues(
-      null,
-      scheduler.getQueueRefresher());
+    refreshQueues(taskTrackerManager.getQueueManager(), null, scheduler);
 
     job1 =
       taskTrackerManager.submitJobAndInit(

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Fri Apr 23 09:10:42
2010
@@ -4298,10 +4298,12 @@ public class JobTracker implements MRCon
   public void refreshQueues() throws IOException{
     LOG.info("Refreshing queue information. requested by : " + 
              UserGroupInformation.getCurrentUser().getShortUserName());
-    this.queueManager.refreshQueues(new Configuration(this.conf),
-        taskScheduler.getQueueRefresher());
+    synchronized (taskScheduler) {
+      queueManager.refreshQueues(new Configuration(this.conf), taskScheduler
+          .getQueueRefresher());
+    }
   }
-
+  
   private void initializeTaskMemoryRelatedConfig() {
     memSizeForMapSlotOnJT =
         JobConf.normalizeMemoryConfigValue(conf.getLong(

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskScheduler.java Fri Apr 23
09:10:42 2010
@@ -99,6 +99,9 @@ abstract class TaskScheduler implements 
    * {@link QueueManager} whenever it gets a request from an administrator to
    * refresh its own queue-configuration. This method has a documented contract
    * between the {@link QueueManager} and the {@link TaskScheduler}.
+   * 
+   * Before calling QueueRefresher, the caller must hold the lock to the
+   * corresponding {@link TaskScheduler} (generally in the {@link JobTracker}).
    */
   abstract class QueueRefresher {
 

Modified: hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml?rev=937201&r1=937200&r2=937201&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml Fri Apr 23 09:10:42 2010
@@ -360,4 +360,24 @@
       <Method name="getHexDigit"/>
       <Bug pattern="ICAST_QUESTIONABLE_UNSIGNED_RIGHT_SHIFT" />
      </Match>
+
+     <!-- 
+	   False positive of FindBugs complaining that initializationPoller and
+	   started were accessed through
+	   CapacitySchedulerQueueRefresher.refreshQueues without explicitly holding
+	   the lock of CapacityTaskScheduler. The lock is in fact acquired by
+	   JobTracker prior to calling QueueManager.refereshQueues - this is
+	   necessary to ensure the order of locking (TaskScheduler ->
+	   QueueManager).
+     --> 
+    <Match>
+       <Class name="org.apache.hadoop.mapred.CapacityTaskScheduler" />
+       <Field name="initializationPoller" />
+       <Bug pattern="IS2_INCONSISTENT_SYNC" />
+    </Match>
+    <Match>
+       <Class name="org.apache.hadoop.mapred.CapacityTaskScheduler" />
+       <Field name="started" />
+       <Bug pattern="IS2_INCONSISTENT_SYNC" />
+    </Match>
  </FindBugsFilter>



Mime
View raw message