hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yhema...@apache.org
Subject svn commit: r788036 [3/3] - in /hadoop/mapreduce/trunk: ./ src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/ src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/ src/contrib/dynamic-scheduler/src/java/org/apache/hadoop/mapr...
Date Wed, 24 Jun 2009 14:22:15 GMT
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
Wed Jun 24 14:22:13 2009
@@ -28,6 +28,7 @@
 
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
 
 public class TestJobQueueTaskScheduler extends TestCase {
   
@@ -76,7 +77,7 @@
     public Task obtainNewMapTask(final TaskTrackerStatus tts, int clusterSize,
         int ignored) throws IOException {
       TaskAttemptID attemptId = getTaskAttemptID(TaskType.MAP);
-      Task task = new MapTask("", attemptId, 0, "", new BytesWritable()) {
+      Task task = new MapTask("", attemptId, 0, "", new BytesWritable(), 1) {
         @Override
         public String toString() {
           return String.format("%s on %s", getTaskID(), tts.getTrackerName());
@@ -91,7 +92,7 @@
     public Task obtainNewReduceTask(final TaskTrackerStatus tts,
         int clusterSize, int ignored) throws IOException {
       TaskAttemptID attemptId = getTaskAttemptID(TaskType.REDUCE);
-      Task task = new ReduceTask("", attemptId, 0, 10) {
+      Task task = new ReduceTask("", attemptId, 0, 10, 1) {
         @Override
         public String toString() {
           return String.format("%s on %s", getTaskID(), tts.getTrackerName());
@@ -119,18 +120,26 @@
       new ArrayList<JobInProgressListener>();
     QueueManager queueManager;
     
-    private Map<String, TaskTrackerStatus> trackers =
-      new HashMap<String, TaskTrackerStatus>();
+    private Map<String, TaskTracker> trackers =
+      new HashMap<String, TaskTracker>();
 
     public FakeTaskTrackerManager() {
       JobConf conf = new JobConf();
       queueManager = new QueueManager(conf);
-      trackers.put("tt1", new TaskTrackerStatus("tt1", "tt1.host", 1,
-                   new ArrayList<TaskStatus>(), 0,
-                   maxMapTasksPerTracker, maxReduceTasksPerTracker));
-      trackers.put("tt2", new TaskTrackerStatus("tt2", "tt2.host", 2,
-                   new ArrayList<TaskStatus>(), 0,
-                   maxMapTasksPerTracker, maxReduceTasksPerTracker));
+      
+      TaskTracker tt1 = new TaskTracker("tt1");
+      tt1.setStatus(new TaskTrackerStatus("tt1", "tt1.host", 1,
+                                          new ArrayList<TaskStatus>(), 0,
+                                          maxMapTasksPerTracker, 
+                                          maxReduceTasksPerTracker));
+      trackers.put("tt1", tt1);
+      
+      TaskTracker tt2 = new TaskTracker("tt2");
+      tt2.setStatus(new TaskTrackerStatus("tt2", "tt2.host", 2,
+                                          new ArrayList<TaskStatus>(), 0,
+                                          maxMapTasksPerTracker, 
+                                          maxReduceTasksPerTracker));
+      trackers.put("tt2", tt2);
     }
     
     @Override
@@ -151,7 +160,11 @@
 
     @Override
     public Collection<TaskTrackerStatus> taskTrackers() {
-      return trackers.values();
+      List<TaskTrackerStatus> statuses = new ArrayList<TaskTrackerStatus>();
+      for (TaskTracker tt : trackers.values()) {
+        statuses.add(tt.getStatus());
+      }
+      return statuses;
     }
 
 
@@ -193,7 +206,7 @@
       }
     }
     
-    public TaskTrackerStatus getTaskTracker(String trackerID) {
+    public TaskTracker getTaskTracker(String trackerID) {
       return trackers.get(trackerID);
     }
     
@@ -210,7 +223,7 @@
         }
       };
       status.setRunState(TaskStatus.State.RUNNING);
-      trackers.get(taskTrackerName).getTaskReports().add(status);
+      trackers.get(taskTrackerName).getStatus().getTaskReports().add(status);
     }
     
   }
@@ -286,14 +299,14 @@
     checkAssignment(scheduler, tracker(taskTrackerManager, "tt2"), new String[] {});
   }
 
-  static TaskTrackerStatus tracker(FakeTaskTrackerManager taskTrackerManager,
+  static TaskTracker tracker(FakeTaskTrackerManager taskTrackerManager,
                                       String taskTrackerName) {
     return taskTrackerManager.getTaskTracker(taskTrackerName);
   }
   
-  static void checkAssignment(TaskScheduler scheduler, TaskTrackerStatus tts,
+  static void checkAssignment(TaskScheduler scheduler, TaskTracker taskTracker,
       String[] expectedTaskStrings) throws IOException {
-    List<Task> tasks = scheduler.assignTasks(tts);
+    List<Task> tasks = scheduler.assignTasks(taskTracker);
     assertNotNull(tasks);
     assertEquals(expectedTaskStrings.length, tasks.size());
     for (int i=0; i < expectedTaskStrings.length; ++i) {

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerSafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerSafeMode.java?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerSafeMode.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerSafeMode.java
Wed Jun 24 14:22:13 2009
@@ -194,7 +194,7 @@
     long jobtrackerRecoveryFinishTime = 
       jobtracker.getStartTime() + jobtracker.getRecoveryDuration();
     for (String trackerName : present) {
-      TaskTrackerStatus status = jobtracker.getTaskTracker(trackerName);
+      TaskTrackerStatus status = jobtracker.getTaskTrackerStatus(trackerName);
       // check if the status is present and also the tracker has contacted back
       // after restart
       if (status == null 
@@ -203,7 +203,7 @@
       }
     }
     for (String trackerName : absent) {
-      TaskTrackerStatus status = jobtracker.getTaskTracker(trackerName);
+      TaskTrackerStatus status = jobtracker.getTaskTrackerStatus(trackerName);
       // check if the status is still present
       if ( status != null) {
         return false;

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestResourceEstimation.java
Wed Jun 24 14:22:13 2009
@@ -46,7 +46,7 @@
       ts.setOutputSize(singleMapOutputSize);
       RawSplit split = new RawSplit();
       split.setDataLength(0);
-      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
+      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1);
       re.updateWithCompletedTask(ts, tip);
     }
     assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
@@ -81,7 +81,7 @@
       ts.setOutputSize(singleMapOutputSize);
       RawSplit split = new RawSplit();
       split.setDataLength(singleMapInputSize);
-      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
+      TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1);
       re.updateWithCompletedTask(ts, tip);
     }
     
@@ -93,7 +93,7 @@
     ts.setOutputSize(singleMapOutputSize);
     RawSplit split = new RawSplit();
     split.setDataLength(0);
-    TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0);
+    TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1);
     re.updateWithCompletedTask(ts, tip);
     
     long expectedTotalMapOutSize = (singleMapOutputSize*11) * 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSpeculativeExecution.java
Wed Jun 24 14:22:13 2009
@@ -105,14 +105,14 @@
         JobClient.RawSplit split = new JobClient.RawSplit();
         split.setLocations(new String[0]);
         maps[i] = new TaskInProgress(getJobID(), "test", 
-            split, jobtracker, getJobConf(), this, i);
+            split, jobtracker, getJobConf(), this, i, 1);
         nonLocalMaps.add(maps[i]);
       }
       reduces = new TaskInProgress[numReduceTasks];
       for (int i = 0; i < numReduceTasks; i++) {
         reduces[i] = new TaskInProgress(getJobID(), "test", 
                                         numMapTasks, i, 
-                                        jobtracker, getJobConf(), this);
+                                        jobtracker, getJobConf(), this, 1);
         nonRunningReduces.add(reduces[i]);
       }
     }
@@ -165,7 +165,7 @@
     public void finishTask(TaskAttemptID taskId) {
       TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
       TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, 
-          1.0f, TaskStatus.State.SUCCEEDED, "", "", tip.machineWhereTaskRan(taskId), 
+          1.0f, 1, TaskStatus.State.SUCCEEDED, "", "", tip.machineWhereTaskRan(taskId), 
           tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters());
       updateTaskStatus(tip, status);
     }
@@ -174,14 +174,14 @@
       addRunningTaskToTIP(tip, taskId, new TaskTrackerStatus(taskTracker,
           JobInProgress.convertTrackerNameToHostName(taskTracker)), true);
       TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, 
-          0.0f, TaskStatus.State.RUNNING, "", "", taskTracker,
+          0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker,
           tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters());
       updateTaskStatus(tip, status);
     }
     public void progressMade(TaskAttemptID taskId, float progress) {
       TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
       TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId, 
-          progress, TaskStatus.State.RUNNING, "", "", tip.machineWhereTaskRan(taskId), 
+          progress, 1, TaskStatus.State.RUNNING, "", "", tip.machineWhereTaskRan(taskId),

           tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters());
       updateTaskStatus(tip, status);
     }
@@ -345,4 +345,4 @@
     }
     return i;
   }
-}
\ No newline at end of file
+}

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTTMemoryReporting.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTTMemoryReporting.java?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTTMemoryReporting.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTTMemoryReporting.java
Wed Jun 24 14:22:13 2009
@@ -26,6 +26,7 @@
 import org.apache.hadoop.util.LinuxMemoryCalculatorPlugin;
 import org.apache.hadoop.util.MemoryCalculatorPlugin;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
 
 import junit.framework.TestCase;
 
@@ -66,9 +67,9 @@
     }
     
     @Override
-    public List<Task> assignTasks(TaskTrackerStatus status)
+    public List<Task> assignTasks(TaskTracker taskTracker)
         throws IOException {
-
+      TaskTrackerStatus status = taskTracker.getStatus();
       long totalVirtualMemoryOnTT =
           getConf().getLong("totalVmemOnTT", JobConf.DISABLED_MEMORY_LIMIT);
       long totalPhysicalMemoryOnTT =
@@ -111,7 +112,7 @@
           || reduceSlotMemorySize != reportedReduceSlotMemorySize) {
         hasPassed = false;
       }
-      return super.assignTasks(status);
+      return super.assignTasks(taskTracker);
     }
   }
 
@@ -126,7 +127,7 @@
     try {
       // Memory values are disabled by default.
       conf.setClass(
-          TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
+          org.apache.hadoop.mapred.TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
           DummyMemoryCalculatorPlugin.class, MemoryCalculatorPlugin.class);
       setUpCluster(conf);
       runSleepJob(miniMRCluster.createJobConf());
@@ -150,7 +151,7 @@
     conf.setLong("reduceSlotMemorySize", 1 * 1024L);
 
     conf.setClass(
-        TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
+        org.apache.hadoop.mapred.TaskTracker.MAPRED_TASKTRACKER_MEMORY_CALCULATOR_PLUGIN_PROPERTY,
         DummyMemoryCalculatorPlugin.class, MemoryCalculatorPlugin.class);
     conf.setLong(DummyMemoryCalculatorPlugin.MAXVMEM_TESTING_PROPERTY,
         4 * 1024 * 1024 * 1024L);

Modified: hadoop/mapreduce/trunk/src/webapps/job/jobfailures.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobfailures.jsp?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/jobfailures.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/jobfailures.jsp Wed Jun 24 14:22:13 2009
@@ -45,7 +45,7 @@
       if ((failState == null && (taskState == TaskStatus.State.FAILED || 
           taskState == TaskStatus.State.KILLED)) || taskState == failState) {
         String taskTrackerName = statuses[i].getTaskTracker();
-        TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
+        TaskTrackerStatus taskTracker = tracker.getTaskTrackerStatus(taskTrackerName);
         out.print("<tr><td>" + statuses[i].getTaskID() +
                   "</td><td><a href=\"taskdetails.jsp?jobid="+ jobId + 
                   "&tipid=" + tipId + "\">" + tipId +

Modified: hadoop/mapreduce/trunk/src/webapps/job/machines.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/machines.jsp?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/machines.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/machines.jsp Wed Jun 24 14:22:13 2009
@@ -82,8 +82,8 @@
         out.print(tt.getHost() + ":" + tt.getHttpPort() + "/\">");
         out.print(tt.getTrackerName() + "</a></td><td>");
         out.print(tt.getHost() + "</td><td>" + numCurTasks +
-                  "</td><td>" + tt.getMaxMapTasks() +
-                  "</td><td>" + tt.getMaxReduceTasks() + 
+                  "</td><td>" + tt.getMaxMapSlots() +
+                  "</td><td>" + tt.getMaxReduceSlots() + 
                   "</td><td>" + numFailures + 
                   "</td><td>" + sinceHeartbeat + "</td></tr>\n");
       }

Modified: hadoop/mapreduce/trunk/src/webapps/job/taskdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/taskdetails.jsp?rev=788036&r1=788035&r2=788036&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/taskdetails.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/taskdetails.jsp Wed Jun 24 14:22:13 2009
@@ -142,7 +142,7 @@
     for (int i = 0; i < ts.length; i++) {
       TaskStatus status = ts[i];
       String taskTrackerName = status.getTaskTracker();
-      TaskTrackerStatus taskTracker = tracker.getTaskTracker(taskTrackerName);
+      TaskTrackerStatus taskTracker = tracker.getTaskTrackerStatus(taskTrackerName);
       out.print("<tr><td>" + status.getTaskID() + "</td>");
       String taskAttemptTracker = null;
       String cleanupTrackerName = null;
@@ -151,7 +151,7 @@
       boolean hasCleanupAttempt = false;
       if (tip != null && tip.isCleanupAttempt(status.getTaskID())) {
         cleanupTrackerName = tip.machineWhereCleanupRan(status.getTaskID());
-        cleanupTracker = tracker.getTaskTracker(cleanupTrackerName);
+        cleanupTracker = tracker.getTaskTrackerStatus(cleanupTrackerName);
         if (cleanupTracker != null) {
           cleanupAttemptTracker = "http://" + cleanupTracker.getHost() + ":"
             + cleanupTracker.getHttpPort();



Mime
View raw message