hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079223 - in /hadoop/mapreduce/branches/yahoo-merge/src: java/org/apache/hadoop/mapred/ java/org/apache/hadoop/mapreduce/ test/mapred/org/apache/hadoop/mapreduce/
Date Tue, 08 Mar 2011 05:57:43 GMT
Author: omalley
Date: Tue Mar  8 05:57:43 2011
New Revision: 1079223

URL: http://svn.apache.org/viewvc?rev=1079223&view=rev
Log:
commit ff4e51b6ddb6c3a534f01ab8835289475baff26d
Author: Greg Roelofs <roelofs@yahoo-inc.com>
Date:   Fri Dec 17 17:22:53 2010 -0800

    Removal of all debug noise (and one trivial error-message formatting fix
    accidentally overlooked in last commit).

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/DefaultTaskController.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskInProgress.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskStatus.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMRJobClient.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/DefaultTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/DefaultTaskController.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/DefaultTaskController.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/DefaultTaskController.java
Tue Mar  8 05:57:43 2011
@@ -71,7 +71,6 @@ public class DefaultTaskController exten
                                   env.workDir, env.env);
     // set the ShellCommandExecutor for later use.
     context.shExec = shexec;
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  DefaultTaskController ctor: about to call ShellCommandExecutor's execute() method");
     shexec.execute();
   }
     
@@ -85,7 +84,6 @@ System.out.println("GRR DEBUG (" + Strin
     // The default task controller does not need to set up
     // any permissions for proper execution.
     // So this is a dummy method.
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  DefaultTaskController initializeTask(): nothing to do here");
     return;
   }
 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
Tue Mar  8 05:57:43 2011
@@ -1163,7 +1163,6 @@ public class JobInProgress {
   public synchronized void updateTaskStatus(TaskInProgress tip, 
                                             TaskStatus status) {
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress updateTaskStatus() starting with TIP " + tip.getTIPId() + " and TaskStatus
" + status + " (phase = " + status.getPhase() + ", state = " + status.getStateString() + ",
progress = " + status.getProgress() + ", runState = " + status.getRunState() + ", TT = " +
status.getTaskTracker() + ", diag = " + status.getDiagnosticInfo() + ")");
     double oldProgress = tip.getProgress();   // save old progress
     boolean wasRunning = tip.isRunning();
     boolean wasComplete = tip.isComplete();
@@ -1198,7 +1197,6 @@ System.out.println("GRR DEBUG (" + Strin
     }
     
     boolean change = tip.updateStatus(status);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress updateTaskStatus(): just called TIP updateStatus() with status " + status
+ " (phase = " + status.getPhase() + "); returned change = " + change);
     if (change) {
       TaskStatus.State state = status.getRunState();
       // get the TaskTrackerStatus where the task ran 
@@ -1243,7 +1241,6 @@ System.out.println("GRR DEBUG (" + Strin
         return;
       } else if (state == TaskStatus.State.FAILED_UNCLEAN ||
                  state == TaskStatus.State.KILLED_UNCLEAN) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress updateTaskStatus(): task state = *_UNCLEAN; calling TIP incompleteSubTask()
and adding cleanup task " + taskid);
         tip.incompleteSubTask(taskid, this.status);
         // add this task, to be rescheduled as cleanup attempt
         if (tip.isMapTask()) {
@@ -1257,7 +1254,6 @@ System.out.println("GRR DEBUG (" + Strin
       //For a failed task update the JT datastructures. 
       else if (state == TaskStatus.State.FAILED ||
                state == TaskStatus.State.KILLED) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress updateTaskStatus(): task state = plain FAILED/KILLED; about to call failedTask()
and make new TaskCompletionEvent");
         // Get the event number for the (possibly) previously successful
         // task. If there exists one, then set that status to OBSOLETE 
         int eventNumber;
@@ -2731,7 +2727,6 @@ System.out.println("GRR DEBUG (" + Strin
   public synchronized boolean completedTask(TaskInProgress tip, 
                                             TaskStatus status)
   {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress completedTask() starting with TIP " + tip.getTIPId() + " and TaskStatus
" + status);
     TaskAttemptID taskid = status.getTaskID();
     final JobTrackerInstrumentation metrics = jobtracker.getInstrumentation();
         
@@ -2810,7 +2805,6 @@ System.out.println("GRR DEBUG (" + Strin
       killSetupTip(!tip.isMapTask());
       setupComplete();
     } else if (tip.isJobCleanupTask()) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress completedTask(): TIP " + tip.getTIPId() + " is a cleanup TIP");
       // cleanup task has finished. Kill the extra cleanup tip
       if (tip.isMapTask()) {
         // kill the reduce tip
@@ -2822,16 +2816,13 @@ System.out.println("GRR DEBUG (" + Strin
       // The Job is done
       // if the job is failed, then mark the job failed.
       if (jobFailed) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress completedTask(): calling terminateJob() with JobStatus.FAILED");
         terminateJob(JobStatus.FAILED);
       }
       // if the job is killed, then mark the job killed.
       if (jobKilled) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress completedTask(): calling terminateJob() with JobStatus.KILLED");
         terminateJob(JobStatus.KILLED);
       }
       else {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress completedTask(): calling jobComplete() (HUH??)");
         jobComplete();
       }
       // The job has been killed/failed/successful
@@ -2873,7 +2864,6 @@ System.out.println("GRR DEBUG (" + Strin
     if (!jobSetupCleanupNeeded && canLaunchJobCleanupTask()) {
       jobComplete();
     }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress completedTask(): done");
     return true;
   }
 
@@ -3010,11 +3000,9 @@ System.out.println("GRR DEBUG (" + Strin
   }
   
   private synchronized void terminateJob(int jobTerminationState) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress terminateJob() called with jobTerminationState = " + jobTerminationState);
     if ((status.getRunState() == JobStatus.RUNNING) ||
         (status.getRunState() == JobStatus.PREP)) {
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress terminateJob(): runState = " + status.getRunState());
       this.finishTime = JobTracker.getClock().getTime();
       this.status.setMapProgress(1.0f);
       this.status.setReduceProgress(1.0f);
@@ -3051,7 +3039,6 @@ System.out.println("GRR DEBUG (" + Strin
             this.conf, this.status.getJobID());
       }
     }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress terminateJob() done");
   }
 
   /**
@@ -3135,13 +3122,11 @@ System.out.println("GRR DEBUG (" + Strin
     while (!mapCleanupTasks.isEmpty()) {
       taskid = mapCleanupTasks.remove(0);
       tip = maps[taskid.getTaskID().getId()];
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress clearUncleanTasks(): calling updateTaskStatus() on map cleanup task "
+ taskid);
       updateTaskStatus(tip, tip.getTaskStatus(taskid));
     }
     while (!reduceCleanupTasks.isEmpty()) {
       taskid = reduceCleanupTasks.remove(0);
       tip = reduces[taskid.getTaskID().getId()];
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress clearUncleanTasks(): calling updateTaskStatus() on reduce cleanup task
" + taskid);
       updateTaskStatus(tip, tip.getTaskStatus(taskid));
     }
   }
@@ -3425,7 +3410,6 @@ System.out.println("GRR DEBUG (" + Strin
   public synchronized void failedTask(TaskInProgress tip, TaskAttemptID taskid,
       String reason, TaskStatus.Phase phase, TaskStatus.State state, 
                          String trackerName) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress failedTask(): called with phase = " + phase + " (creating new TaskStatus)");
     TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), 
                                                     taskid,
                                                     0.0f,
@@ -3445,7 +3429,6 @@ System.out.println("GRR DEBUG (" + Strin
     status.setStartTime(startTime);
     status.setFinishTime(JobTracker.getClock().getTime());
     boolean wasComplete = tip.isComplete();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress failedTask(): calling updateTaskStatus()");
     updateTaskStatus(tip, status);
     boolean isComplete = tip.isComplete();
     if (wasComplete && !isComplete) { // mark a successful tip as failed

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobTracker.java
Tue Mar  8 05:57:43 2011
@@ -2480,14 +2480,12 @@ public class JobTracker implements MRCon
     // Check for tasks to be killed
     List<TaskTrackerAction> killTasksList = getTasksToKill(trackerName);
     if (killTasksList != null) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker heartbeat(): found non-empty killTasksList");
       actions.addAll(killTasksList);
     }
      
     // Check for jobs to be killed/cleanedup
     List<TaskTrackerAction> killJobsList = getJobsForCleanup(trackerName);
     if (killJobsList != null) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker heartbeat(): found non-empty killJobsList");
       actions.addAll(killJobsList);
     }
 
@@ -2560,7 +2558,6 @@ System.out.println("GRR DEBUG (" + Strin
    */
   boolean updateTaskTrackerStatus(String trackerName,
                                           TaskTrackerStatus status) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskTrackerStatus(): starting");
     TaskTracker tt = getTaskTracker(trackerName);
     TaskTrackerStatus oldStatus = (tt == null) ? null : tt.getStatus();
     if (oldStatus != null) {
@@ -2619,7 +2616,7 @@ System.out.println("GRR DEBUG (" + Strin
       taskTracker.setStatus(status);
       taskTrackers.put(trackerName, taskTracker);
       
-      if (true/* GRR DEBUG LOG.isDebugEnabled()*/) {
+      if (LOG.isDebugEnabled()) {
         int runningMaps = 0, runningReduces = 0;
         int commitPendingMaps = 0, commitPendingReduces = 0;
         int unassignedMaps = 0, unassignedReduces = 0;
@@ -2643,7 +2640,7 @@ System.out.println("GRR DEBUG (" + Strin
             else { ++miscReduces; } 
           }
         }
-        LOG.info/* GRR DEBUG .debug */(trackerName + ": Status -" +
+        LOG.debug(trackerName + ": Status -" +
                   " running(m) = " + runningMaps + 
                   " unassigned(m) = " + unassignedMaps + 
                   " commit_pending(m) = " + commitPendingMaps +
@@ -3955,11 +3952,9 @@ System.out.println("GRR DEBUG (" + Strin
    */
   void updateTaskStatuses(TaskTrackerStatus status) {
     String trackerName = status.getTrackerName();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): starting (TT = " + trackerName + ")");
     for (TaskStatus report : status.getTaskReports()) {
       report.setTaskTracker(trackerName);
       TaskAttemptID taskId = report.getTaskID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): processing TaskStatus for task ID " + taskId + " with
phase (from TT) = " + report.getPhase());
       
       // expire it
       expireLaunchingTasks.removeTask(taskId);
@@ -3973,14 +3968,12 @@ System.out.println("GRR DEBUG (" + Strin
             jobs = new HashSet<JobID>();
             trackerToJobsToCleanup.put(trackerName, jobs);
           }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): adding job ID " + taskId.getJobID() + " to job-cleanup
list");
           jobs.add(taskId.getJobID());
         }
         continue;
       }
       
       if (!job.inited()) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): job " + job + " not initialized; killing attempt");
         // if job is not yet initialized ... kill the attempt
         synchronized (trackerToTasksToCleanup) {
           Set<TaskAttemptID> tasks = trackerToTasksToCleanup.get(trackerName);
@@ -3988,7 +3981,6 @@ System.out.println("GRR DEBUG (" + Strin
             tasks = new HashSet<TaskAttemptID>();
             trackerToTasksToCleanup.put(trackerName, tasks);
           }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): adding task ID " + taskId + " to task-cleanup list");
           tasks.add(taskId);
         }
         continue;
@@ -4003,13 +3995,11 @@ System.out.println("GRR DEBUG (" + Strin
         // or TaskInProgress can modify this object and
         // the changes should not get reflected in TaskTrackerStatus.
         // An old TaskTrackerStatus is used later in countMapTasks, etc.
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): found TIP; calling updateTaskStatus() on " + taskId);
         job.updateTaskStatus(tip, (TaskStatus)report.clone());
         JobStatus newStatus = (JobStatus)job.getStatus().clone();
         
         // Update the listeners if an incomplete job completes
         if (prevStatus.getRunState() != newStatus.getRunState()) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): creating JobStatusChangeEvent");
           JobStatusChangeEvent event = 
             new JobStatusChangeEvent(job, EventType.RUN_STATE_CHANGED, 
                                      prevStatus, newStatus);
@@ -4027,7 +4017,6 @@ System.out.println("GRR DEBUG (" + Strin
           TaskInProgress failedFetchMap = taskidToTIPMap.get(mapTaskId);
           
           if (failedFetchMap != null) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobTracker updateTaskStatuses(): doing 'failed fetch' stuff");
             // Gather information about the map which has to be failed, if need be
             String failedFetchTrackerName = getAssignedTracker(mapTaskId);
             if (failedFetchTrackerName == null) {

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java Tue
Mar  8 05:57:43 2011
@@ -319,7 +319,6 @@ abstract public class Task implements Wr
   }
 
   void setTaskCleanupTask() {
-System.out.println("GRR DEBUG:  Task setTaskCleanupTask(): setting taskCleanup = true (=>
phase = CLEANUP on deserialization!)");
     taskCleanup = true;
   }
 	   
@@ -429,7 +428,6 @@ System.out.println("GRR DEBUG:  Task set
     writeSkipRecs = in.readBoolean();
     taskCleanup = in.readBoolean();
     if (taskCleanup) {
-System.out.println("GRR DEBUG:  Task readFields(): setting phase to CLEANUP");
       setPhase(TaskStatus.Phase.CLEANUP);
     }
     user = Text.readString(in);
@@ -1015,7 +1013,6 @@ System.out.println("GRR DEBUG:  Task rea
       try {
         while (!umbilical.canCommit(taskIdForUmbilical)) {
           try {
-System.out.println("GRR DEBUG:  Task commitAfterApproval(): TT canCommit() returned false;
sleeping 1 sec");
             // FIXME 1:  shouldn't this count down retries, too, in case JT glitched and
no longer knows about us?  (else infinite loop)
             Thread.sleep(1000);  // FIXME 2:  shouldn't hardcoded 1-second sleep instead
correspond to heartbeat interval for task?
           } catch(InterruptedException ie) {
@@ -1023,10 +1020,8 @@ System.out.println("GRR DEBUG:  Task com
           }
           reporter.setProgressFlag();
         }
-System.out.println("GRR DEBUG:  Task commitAfterApproval(): TT canCommit() returned true");
         break;
       } catch (IOException ie) {
-System.out.println("GRR DEBUG:  Task commitAfterApproval(): TT canCommit() threw exception");
         LOG.warn("Failure asking whether task can commit: " +
             StringUtils.stringifyException(ie));
         if (--retries == 0) {
@@ -1044,7 +1039,6 @@ System.out.println("GRR DEBUG:  Task com
   // this is protected (rather than private) solely for UberTask map-only case
   protected void commit(TaskUmbilicalProtocol umbilical,
                         TaskReporter reporter) throws IOException {
-System.out.println("GRR DEBUG:  Task commit(): about to call commitTask()");
     try {
       LOG.info("Task " + taskId + " is allowed to commit now");
       committer.commitTask(taskContext);
@@ -1078,7 +1072,6 @@ System.out.println("GRR DEBUG:  Task com
   void taskCleanup(TaskUmbilicalProtocol umbilical) 
   throws IOException {
     // set phase for this task
-System.out.println("GRR DEBUG:  Task taskCleanup(): setting phase to CLEANUP");
     setPhase(TaskStatus.Phase.CLEANUP);
     getProgress().setStatus("cleanup");
     statusUpdate(umbilical);
@@ -1091,7 +1084,6 @@ System.out.println("GRR DEBUG:  Task tas
                                TaskReporter reporter
                               ) throws IOException, InterruptedException {
     // set phase for this task
-System.out.println("GRR DEBUG:  Task runJobCleanupTask(): setting phase to CLEANUP");
     setPhase(TaskStatus.Phase.CLEANUP);
     getProgress().setStatus("cleanup");
     statusUpdate(umbilical);

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskInProgress.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskInProgress.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskInProgress.java
Tue Mar  8 05:57:43 2011
@@ -781,7 +781,6 @@ class TaskInProgress {
           setProgressRate(currProgRate);
         }
       } else {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskInProgress updateStatus(): calling TaskStatus statusUpdate() for " + taskid + "
with phase " + status.getPhase());
         taskStatuses.get(taskid).statusUpdate(status.getRunState(),
                                               status.getProgress(), status.getStateString(),
status.getPhase(),
                                               status.getFinishTime());

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskStatus.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskStatus.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskStatus.java
Tue Mar  8 05:57:43 2011
@@ -95,9 +95,7 @@ public abstract class TaskStatus impleme
   public TaskStatus(TaskAttemptID taskid, float progress, int numSlots,
                     State runState, String diagnosticInfo,
                     String stateString, String taskTracker,
-                    Phase phase, Counters counters)
-//throws IOException
- {
+                    Phase phase, Counters counters) {
     this.taskid = taskid;
     this.progress = progress;
     this.numSlots = numSlots;
@@ -105,11 +103,9 @@ public abstract class TaskStatus impleme
     setDiagnosticInfo(diagnosticInfo);
     setStateString(stateString);
     this.taskTracker = taskTracker;
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskStatus ctor: creating with runState = " + runState + " and phase = " + phase);
     this.phase = phase;
     this.counters = counters;
     this.includeAllCounters = true;
-//if (runState == TaskStatus.State.FAILED_UNCLEAN && phase == TaskStatus.Phase.MAP)
{ throw new IOException("GRR DEBUG:  TaskStatus ctor:  WHO'S THE LAMER THAT CALLED US???");
}
   }
   
   public TaskAttemptID getTaskID() { return taskid; }
@@ -311,7 +307,6 @@ System.out.println("GRR DEBUG (" + Strin
       }else if (phase == TaskStatus.Phase.REDUCE){
         setSortFinishTime(System.currentTimeMillis());
       }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskStatus setPhase(): changing phase from " + getPhase() + " to " + phase);
       this.phase = phase;
     }
   }
@@ -413,7 +408,6 @@ System.out.println("GRR DEBUG (" + Strin
       this.setFinishTime(status.getFinishTime()); 
     }
     
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskStatus statusUpdate() #2: changing phase from " + getPhase() + " to " + status.getPhase());
     this.phase = status.getPhase();
     this.counters = status.getCounters();
     this.outputSize = status.outputSize;
@@ -439,7 +433,6 @@ System.out.println("GRR DEBUG (" + Strin
     setRunState(runState);
     setProgress(progress);
     setStateString(state);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskStatus statusUpdate() #3: changing phase from " + getPhase() + " to " + phase);
     setPhase(phase);
     if (finishTime > 0) {
       setFinishTime(finishTime); 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskTracker.java
Tue Mar  8 05:57:43 2011
@@ -424,12 +424,10 @@ public class TaskTracker 
       if (!runningJobs.containsKey(jobId)) {
         rJob = new RunningJob(jobId);
         rJob.tasks = new HashSet<TaskInProgress>();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker addTaskToJob(): adding running job " + jobId + " to runningJobs");
         runningJobs.put(jobId, rJob);
       } else {
         rJob = runningJobs.get(jobId);
       }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker addTaskToJob(): adding Task/TIP " + tip.getTask().getTaskID() + " to runningJob
" + jobId);
       synchronized (rJob) {
         rJob.tasks.add(tip);
       }
@@ -443,7 +441,6 @@ System.out.println("GRR DEBUG (" + Strin
       if (rjob == null) {
         LOG.warn("Unknown job " + jobId + " being deleted.");
       } else {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker removeTaskFromJob(): removing Task/TIP " + tip.getTask().getTaskID() +
" from runningJob " + jobId);
         synchronized (rjob) {
           rjob.tasks.remove(tip);
         }
@@ -1673,7 +1670,6 @@ System.out.println("GRR DEBUG (" + Strin
     //
     if (status == null) {
       synchronized (this) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker transmitHeartBeat(): regenerating TaskTrackerStatus");
         status = new TaskTrackerStatus(taskTrackerName, localHostname, 
                                        httpPort, 
                                        cloneAndResetRunningTaskStatuses(
@@ -1741,7 +1737,6 @@ System.out.println("GRR DEBUG (" + Strin
     //
     // Xmit the heartbeat
     //
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker transmitHeartBeat(): sending heartbeat");
     HeartbeatResponse heartbeatResponse = jobClient.heartbeat(status, 
                                                               justStarted,
                                                               justInited,
@@ -1755,7 +1750,6 @@ System.out.println("GRR DEBUG (" + Strin
       
     synchronized (this) {
       for (TaskStatus taskStatus : status.getTaskReports()) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker transmitHeartBeat(): looping over TaskStatuses (task reports): Task/TIP
" + taskStatus.getTaskID() + " runState = " + taskStatus.getRunState() + ", phase = " + taskStatus.getPhase());
         if (taskStatus.getRunState() != TaskStatus.State.RUNNING &&
             taskStatus.getRunState() != TaskStatus.State.UNASSIGNED &&
             taskStatus.getRunState() != TaskStatus.State.COMMIT_PENDING &&
@@ -1770,7 +1764,6 @@ System.out.println("GRR DEBUG (" + Strin
           } catch (Exception e) {
             LOG.warn("Caught: " + StringUtils.stringifyException(e));
           }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker transmitHeartBeat(): removing Task/TIP " + taskStatus.getTaskID() + " from
runningTasks (runState = " + taskStatus.getRunState() + ")");
           runningTasks.remove(taskStatus.getTaskID());
         }
       }
@@ -1996,7 +1989,6 @@ System.out.println("GRR DEBUG (" + Strin
       }
     }
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker addTaskToJob(): removing running job " + jobId + " from runningJobs");
     synchronized(runningJobs) {
       runningJobs.remove(jobId);
     }
@@ -2120,7 +2112,6 @@ System.out.println("GRR DEBUG (" + Strin
         }
       }
     }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker findTaskToKill(): returning TIP " + killMe.getTask().getTaskID());
     return killMe;
   }
 
@@ -2314,7 +2305,6 @@ System.out.println("GRR DEBUG (" + Strin
     LOG.info("LaunchTaskAction (registerTask): " + t.getTaskID() +
              " task's state:" + t.getState());
     TaskInProgress tip = new TaskInProgress(t, this.fConf, launcher);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker registerTask(): adding Task/TIP " + t.getTaskID() + " to runningTasks");
     synchronized (this) {
       tasks.put(t.getTaskID(), tip);
       runningTasks.put(t.getTaskID(), tip);
@@ -2646,7 +2636,6 @@ System.out.println("GRR DEBUG (" + Strin
         return;
       }
       
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker reportProgress(): calling TaskStatus statusUpdate() for " + task.getTaskID()
+ " with phase " + taskStatus.getPhase());
       this.taskStatus.statusUpdate(taskStatus);
       this.lastProgressReport = System.currentTimeMillis();
     }
@@ -3047,7 +3036,6 @@ System.out.println("GRR DEBUG (" + Strin
         taskStatus.setProgress(0.0f);
         reportDiagnosticInfo("Map output lost, rescheduling: " + 
                              failure);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker mapOutputLost(): adding Task/TIP " + task.getTaskID() + " to runningTasks");
         runningTasks.put(task.getTaskID(), this);
         mapTotal++;
         myInstrumentation.statusUpdate(task, taskStatus);
@@ -3425,7 +3413,6 @@ System.out.println("GRR DEBUG (" + Strin
       }
       result.add((TaskStatus)status.clone());
       status.clearStatus();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TaskTracker cloneAndResetRunningTaskStatuses(): Task/TIP " + status.getTaskID() + "
runState = " + status.getRunState() + ", phase = " + status.getPhase());
     }
     return result;
   }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/UberTask.java
Tue Mar  8 05:57:43 2011
@@ -59,7 +59,6 @@ class UberTask extends Task {
   public UberTask() {
     super();
     this.taskStatus = new UberTaskStatus();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask plain ctor finished");
   }
 
   public UberTask(String jobFile, TaskAttemptID taskId, int partition,
@@ -78,7 +77,6 @@ System.out.println("GRR DEBUG (" + Strin
       LOG.debug("UberTask " + getTaskID() + " constructed with " + numMapTasks
                 + " sub-maps and " + numReduceTasks + " sub-reduces");
     }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask normal ctor finished (" + getTaskID() + " constructed with " + numMapTasks
+ " sub-maps and " + numReduceTasks + " sub-reduces)");
   }
 
   @Override
@@ -108,8 +106,6 @@ System.out.println("GRR DEBUG (" + Strin
   throws IOException, ClassNotFoundException, InterruptedException {
     this.umbilical = umbilical;
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask run(): starting");
-
     // set up two-level Progress/phase tree:  getProgress() is root ("uber"),
     // and subtasks' "root node" Progress is second level (will override
     // native one when construct each subtask)
@@ -191,20 +187,16 @@ System.out.println("GRR DEBUG (" + Strin
     // clean up the job (switch phase to "cleanup" and delete staging dir, but
     // do NOT delete temp dir yet)
     if (jobSetupCleanupNeeded) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask run(): about to call runCommitAbortJob() (=> switch phase to 'cleanup'
and delete staging dir but NOT temp dir)");
       runCommitAbortJob(reporter);
     }
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask run(): about to call done() (=> commitTask() or abortTask())");
     // this is where commitTask() (or abortTask()) is called
     done(umbilical, reporter);
 
     // now finish cleaning up the job (delete temp dir:  results are committed)
     if (jobSetupCleanupNeeded) {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask run(): about to call commitJob() (=> delete temp dir)");
       commitJob();
     }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask run(): totally done");
   }
 
   private TaskAttemptID[] createMapIds() {
@@ -439,7 +431,6 @@ System.out.println("GRR DEBUG (" + Strin
         splits[j] = splitIndex;
       }
     }
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  UberTask readFields() finished (" + getTaskID() + " restored with " + numMapTasks +
" sub-maps and " + numReduceTasks + " sub-reduces; phase = " + getPhase() + ")");
   }
 
   /**

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/Job.java Tue
Mar  8 05:57:43 2011
@@ -574,9 +574,7 @@ public class Job extends JobContextImpl 
    */
   public Counters getCounters() 
       throws IOException, InterruptedException {
-System.out.println("GRR DEBUG:  Job getCounters(): about to call ensureState()");
     ensureState(JobState.RUNNING);
-System.out.println("GRR DEBUG:  Job getCounters(): back from ensureState(); about to call
cluster.getClient().getJobCounters()");
     return cluster.getClient().getJobCounters(getJobID());
   }
 
@@ -1118,11 +1116,7 @@ System.out.println("GRR DEBUG:  Job getC
     IntegerRanges reduceRanges = getProfileTaskRange(false);
     int progMonitorPollIntervalMillis = 
       Job.getProgressPollInterval(clientConf);
-//GRR DEBUG ONLY!
-long numIters_GRR_DEBUG = 0;
     while (!isComplete()) {
-++numIters_GRR_DEBUG;
-if (numIters_GRR_DEBUG < 10 || (numIters_GRR_DEBUG % 10) == 0) { System.out.println("GRR
DEBUG:  Job monitorAndPrintJob(): while-loop iteration #" + numIters_GRR_DEBUG); }
       Thread.sleep(progMonitorPollIntervalMillis);
       String report = 
         (" map " + StringUtils.formatPercent(mapProgress(), 0)+
@@ -1137,15 +1131,12 @@ if (numIters_GRR_DEBUG < 10 || (numIters
         getTaskCompletionEvents(eventCounter, 10); 
       eventCounter += events.length;
       printTaskEvents(events, filter, profiling, mapRanges, reduceRanges);
-if (numIters_GRR_DEBUG > 40) { throw new InterruptedException(); } // then check TEST*.txt
file and/or build/test/logs/.../* for additional logged clues
     }
     LOG.info("Job complete: " + jobId);
     Counters counters = getCounters();
-System.out.println("GRR DEBUG:  Job monitorAndPrintJob(): back from getCounters()");
     if (counters != null) {
       LOG.info(counters.toString());
     }
-System.out.println("GRR DEBUG:  Job monitorAndPrintJob(): returning " + isSuccessful() +
" (status.getState() = " + status.getState() + ")");
     return isSuccessful();
   }
 
@@ -1288,7 +1279,7 @@ System.out.println("GRR DEBUG:  Job moni
         input.close();
       }
     } catch(IOException ioe) {
-      LOG.warn("Error reading task output" + ioe.getMessage()); 
+      LOG.warn("Error reading task output: " + ioe.getMessage()); 
     }
   }
   

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMRJobClient.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMRJobClient.java?rev=1079223&r1=1079222&r2=1079223&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMRJobClient.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestMRJobClient.java
Tue Mar  8 05:57:43 2011
@@ -64,10 +64,8 @@ public class TestMRJobClient extends Clu
     }
   }
 
-/* GRR TEMP HACK
   @Test
   public void testJobClient() throws Exception {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testJobClient(): starting");
     Configuration conf = createJobConf();
     Job job = runJob(conf);
     String jobId = job.getJobID().toString();
@@ -79,7 +77,6 @@ System.out.println("GRR DEBUG (" + Strin
   @Test
   public void testGetCounter(String jobId,
       Configuration conf) throws Exception {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testGetCounter(): starting");
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     int exitCode = runTool(conf, createJobClient(),
         new String[] { "-counter", jobId,
@@ -92,7 +89,6 @@ System.out.println("GRR DEBUG (" + Strin
   @Test
   public void testJobList(String jobId,
       Configuration conf) throws Exception {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testJobList(): starting");
     verifyJobPriority(jobId, "HIGH", conf, createJobClient());
   }
 
@@ -120,18 +116,15 @@ System.out.println("GRR DEBUG (" + Strin
   @Test
   public void testChangingJobPriority(String jobId, Configuration conf)
       throws Exception {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testChangingJobPriority(): starting");
     int exitCode = runTool(conf, createJobClient(),
         new String[] { "-set-priority", jobId, "VERY_LOW" },
         new ByteArrayOutputStream());
     assertEquals("Exit code", 0, exitCode);
     verifyJobPriority(jobId, "VERY_LOW", conf, createJobClient());
   }
-END GRR TEMP HACK */
 
   @Test
   public void testMissingProfileOutput() throws Exception {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testMissingProfileOutput(): starting");
     Configuration conf = createJobConf();
     final String input = "hello1\n";
 
@@ -149,9 +142,7 @@ System.out.println("GRR DEBUG (" + Strin
     job.setMaxReduceAttempts(1);
     job.setJobSetupCleanupNeeded(false);
     job.waitForCompletion(true);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testMissingProfileOutput(): done with job 1; BAILING (TEMP HACK)");
if (job != null) return;
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testMissingProfileOutput(): done with job 1; about to start job 2");
     // Run another job with a (valid) hprof agentlib param; verify
     // that the HTTP fetch (of task reports) works here.
     Job job2 = MapReduceTestUtil.createJob(conf,
@@ -168,7 +159,6 @@ System.out.println("GRR DEBUG (" + Strin
     job2.setJobSetupCleanupNeeded(false);
     job2.waitForCompletion(true);
 
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestMRJobClient testMissingProfileOutput(): done with job 2; getting final task reports,
etc.");
     // Find the first map task, verify that we got its profile output file.
     TaskReport [] reports = job2.getTaskReports(TaskType.MAP);
     assertTrue("No task reports found!", reports.length > 0);



Mime
View raw message