hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079218 - in /hadoop/mapreduce/branches/yahoo-merge/src: contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/ java/org/apache/hadoop/mapred/ test/mapred/org/apache/hadoop/mapred/ webapps/job/
Date Tue, 08 Mar 2011 05:57:13 GMT
Author: omalley
Date: Tue Mar  8 05:57:12 2011
New Revision: 1079218

URL: http://svn.apache.org/viewvc?rev=1079218&view=rev
Log:
commit 839669d69fbc0528ac1855611c659214a5bfd19c
Author: Greg Roelofs <roelofs@yahoo-inc.com>
Date:   Tue Dec 14 18:44:39 2010 -0800

    Added fix for failed/killed-uberjob cleanup; removed redundant JIP accessors
    for speculation; explicitly forced speculation off for uber mode; fixed four
    unit tests; renamed JIP getUberMode() accessor to more expected isUber();
    fixed order of expected/actual values in TestBadRecords asserts.  (Still
    lots of debug junk--will remove in next commit.)

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
    hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp

Modified: hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
Tue Mar  8 05:57:12 2011
@@ -630,9 +630,8 @@ class CapacityTaskScheduler extends Task
     boolean hasSpeculativeTask(JobInProgress job, TaskTrackerStatus tts) {
       //Check if job supports speculative map execution first then
       //check if job has speculative maps.
-      return (job.getMapSpeculativeExecution()) && (
-          hasSpeculativeTask(job.getTasks(TaskType.MAP),
-                             tts));
+      return (job.hasSpeculativeMaps() &&
+              hasSpeculativeTask(job.getTasks(TaskType.MAP), tts));
     }
 
   }
@@ -677,9 +676,8 @@ class CapacityTaskScheduler extends Task
     boolean hasSpeculativeTask(JobInProgress job, TaskTrackerStatus tts) {
       //check if the job supports reduce speculative execution first then
       //check if the job has speculative tasks.
-      return (job.getReduceSpeculativeExecution()) && (
-          hasSpeculativeTask(job.getTasks(TaskType.REDUCE),
-                             tts));
+      return (job.hasSpeculativeReduces() &&
+              hasSpeculativeTask(job.getTasks(TaskType.REDUCE), tts));
     }
 
   }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
Tue Mar  8 05:57:12 2011
@@ -141,6 +141,8 @@ public class JobInProgress {
   // uber internals tracked here to avoid need for new TIP get-accessors
   private int uberMapTasks = 0;
   private int uberReduceTasks = 0;
+  private boolean uberSetupCleanupNeeded = false;
+  private boolean uberMode = false;
 
   int mapFailuresPercent = 0;
   int reduceFailuresPercent = 0;
@@ -151,7 +153,6 @@ public class JobInProgress {
   private volatile boolean jobKilled = false;
   private volatile boolean jobFailed = false;
   private /* final */ boolean jobSetupCleanupNeeded;
-  private boolean uberMode = false;
 
   JobPriority priority = JobPriority.NORMAL;
   protected JobTracker jobtracker;
@@ -575,18 +576,10 @@ public class JobInProgress {
   /**
    * Was the job small enough to uberize?
    */
-  public boolean getUberMode() {
+  public boolean isUber() {
     return uberMode;
   }
 
-  boolean getMapSpeculativeExecution() {
-    return hasSpeculativeMaps;
-  }
-  
-  boolean getReduceSpeculativeExecution() {
-    return hasSpeculativeReduces;
-  }
-  
   long getMemoryForMapTask() {
     return memoryPerMap;
   }
@@ -696,14 +689,19 @@ public class JobInProgress {
             || sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT);
 
     if (uberMode) {
-      // save internal details for UI
+      // save internal details for UI and abort-cleanup
       uberMapTasks = numMapTasks;
       uberReduceTasks = numReduceTasks;
+      uberSetupCleanupNeeded = jobSetupCleanupNeeded;
 
-      // this method modifies numMapTasks (-> 0), numReduceTasks (-> 1), and
-      // jobSetupCleanupNeeded (-> false)  [and actually creates a TIP, not a
-      // true Task; latter is created in TaskInProgress's addRunningTask()
-      // method]
+      // disable speculation:  makes no sense to speculate an entire job
+      hasSpeculativeMaps = hasSpeculativeReduces = false;
+
+      // This method modifies numMapTasks (-> 0) and numReduceTasks (-> 1)
+      // [and actually creates a TIP, not a true Task; latter is created in
+      // TaskInProgress's addRunningTask() method after JT/scheduler logic
+      // triggers a LaunchTaskAction].  jobSetupCleanupNeeded is left alone
+      // until initSetupCleanupTasks() below.
       createUberTask(jobFile.toString(), taskSplitMetaInfo);
     }
 
@@ -739,12 +737,10 @@ public class JobInProgress {
 
     // Creates setup[] and cleanup[] arrays of TIPs, with TaskID indices above
     // regular map and reduce values.  In uber mode, UberTask itself handles
-    // setup and cleanup duties, and jobSetupCleanupNeeded is forced false.
-    // But we skip initSetupCleanupTasks() call in that case anyway to avoid
-    // a potentially misleading log msg.
-    if (!uberMode) {
-      initSetupCleanupTasks(jobFile.toString());
-    }
+    // setup and cleanup duties, and jobSetupCleanupNeeded is forced false,
+    // but only after creating cleanup TIPs (in case job fails or is killed =>
+    // still want abortTask() cleanup [possibly custom] to happen).
+    initSetupCleanupTasks(jobFile.toString());
 
     synchronized(jobInitKillStatus){
       jobInitKillStatus.initDone = true;
@@ -780,6 +776,7 @@ public class JobInProgress {
     return maps.length == 0 && reduces.length == 0 && !jobSetupCleanupNeeded;
   }
   
+//GRR FIXME?  who calls this, and what should it return for ubertasks?
   synchronized boolean isSetupCleanupRequired() {
    return jobSetupCleanupNeeded;
   }
@@ -864,7 +861,6 @@ public class JobInProgress {
     nonRunningReduces.add(reduces[0]);  // note:  no map analogue
     numMapTasks = 0;
     numReduceTasks = 1;
-    jobSetupCleanupNeeded = false;
     LOG.info("Input size for job " + jobId + " = " + inputLength
         + ". Number of splits = " + splits.length + ". UberTasking!");
   }
@@ -890,6 +886,16 @@ public class JobInProgress {
                        numReduceTasks, jobtracker, conf, this, 1);
     cleanup[1].setJobCleanupTask();
 
+    if (uberMode) {
+      // ubertasks handle setup internally (as well as cleanup in the normal
+      // case), so henceforth we pretend that setup and cleanup aren't needed
+      // --unless/until job fails or is killed, in which case a separate
+      // cleanup task will be triggered
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress initSetupCleanupTasks(): forcing jobSetupCleanupNeeded to false");
+      jobSetupCleanupNeeded = false;
+      return;
+    }
+
     // Create two setup tips, one map and one reduce.  Only one will be used.
     setup = new TaskInProgress[2];
 
@@ -1492,16 +1498,19 @@ public class JobInProgress {
                                              int numUniqueHosts,
                                              boolean isMapSlot
                                             ) throws IOException {
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): starting");
     if(!tasksInited.get() || !jobSetupCleanupNeeded) {
       return null;
     }
     
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): about to enter locked section");
     synchronized(this) {
       if (!canLaunchJobCleanupTask()) {
         return null;
       }
       
       String taskTracker = tts.getTrackerName();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): taskTracker = " + taskTracker);
       // Update the last-known clusterSize
       this.clusterSize = clusterSize;
       if (!shouldRunOnTaskTracker(taskTracker)) {
@@ -1516,12 +1525,14 @@ public class JobInProgress {
       }
       TaskInProgress tip = findTaskFromList(cleanupTaskList,
                              tts, numUniqueHosts, false);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): found cleanup TIP " + tip);
       if (tip == null) {
         return null;
       }
       
       // Now launch the cleanupTask
       Task result = tip.getTaskToRun(tts.getTrackerName());
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): launching cleanup Task " + result);
       if (result != null) {
         addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
         if (jobFailed) {
@@ -1583,10 +1594,13 @@ public class JobInProgress {
                                              int numUniqueHosts,
                                              boolean isMapSlot
                                             ) throws IOException {
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobSetupTask(): starting");
     if(!tasksInited.get() || !jobSetupCleanupNeeded) {
       return null;
     }
+//GRR FIXME:  need special protection here (since no setup TIPs for ubermode)?  may need
to split jobSetupCleanupNeeded into setup and cleanup halves...
     
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobSetupTask(): about to enter locked section");
     synchronized(this) {
       if (!canLaunchSetupTask()) {
         return null;
@@ -1612,6 +1626,7 @@ public class JobInProgress {
       
       // Now launch the setupTask
       Task result = tip.getTaskToRun(tts.getTrackerName());
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobSetupTask(): launching setup Task " + result);
       if (result != null) {
         addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
       }
@@ -2239,7 +2254,7 @@ public class JobInProgress {
       }
     }
 
-  return slowestTIP;
+    return slowestTIP;
   }
 
   /**
@@ -3043,12 +3058,19 @@ public class JobInProgress {
    * @param jobTerminationState job termination state
    */
   private synchronized void terminate(int jobTerminationState) {
-    if(!tasksInited.get()) {
-    	//init could not be done, we just terminate directly.
+    if (!tasksInited.get()) {
+      // init could not be done, we just terminate directly.
       terminateJob(jobTerminationState);
       return;
     }
 
+    if (uberMode) {
+      // restore setup/cleanup status so separate cleanup task will be launched
+      // (see obtainJobCleanupTask())
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress terminate(): restoring jobSetupCleanupNeeded to " + jobSetupCleanupNeeded);
+      jobSetupCleanupNeeded = uberSetupCleanupNeeded;
+    }
+
     if ((status.getRunState() == JobStatus.RUNNING) ||
          (status.getRunState() == JobStatus.PREP)) {
       LOG.info("Killing job '" + this.status.getJobID() + "'");

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestBadRecords.java
Tue Mar  8 05:57:12 2011
@@ -73,6 +73,7 @@ public class TestBadRecords extends Clus
     conf.setNumMapTasks(1);
     conf.setNumReduceTasks(1);
     conf.setInt(JobContext.TASK_TIMEOUT, 30*1000);
+    conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
     SkipBadRecords.setMapperMaxSkipRecords(conf, Long.MAX_VALUE);
     SkipBadRecords.setReducerMaxSkipGroups(conf, Long.MAX_VALUE);
     
@@ -112,28 +113,28 @@ public class TestBadRecords extends Clus
     LOG.info(runningJob.getCounters().toString());
     assertTrue(runningJob.isSuccessful());
     
-    //validate counters
+    // validate counters (1st assert arg is _expected_ value!)
     Counters counters = runningJob.getCounters();
-    assertEquals(counters.findCounter(TaskCounter.MAP_SKIPPED_RECORDS).
-        getCounter(),mapperBadRecords.size());
+    assertEquals(mapperBadRecords.size(),
+        counters.findCounter(TaskCounter.MAP_SKIPPED_RECORDS).getCounter());
     
     int mapRecs = input.size() - mapperBadRecords.size();
-    assertEquals(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).
-        getCounter(),mapRecs);
-    assertEquals(counters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).
-        getCounter(),mapRecs);
+    assertEquals(mapRecs,
+        counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getCounter());
+    assertEquals(mapRecs,
+        counters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getCounter());
     
     int redRecs = mapRecs - redBadRecords.size();
-    assertEquals(counters.findCounter(TaskCounter.REDUCE_SKIPPED_RECORDS).
-        getCounter(),redBadRecords.size());
-    assertEquals(counters.findCounter(TaskCounter.REDUCE_SKIPPED_GROUPS).
-        getCounter(),redBadRecords.size());
-    assertEquals(counters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).
-        getCounter(),redRecs);
-    assertEquals(counters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).
-        getCounter(),redRecs);
-    assertEquals(counters.findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS).
-        getCounter(),redRecs);
+    assertEquals(redBadRecords.size(),
+        counters.findCounter(TaskCounter.REDUCE_SKIPPED_RECORDS).getCounter());
+    assertEquals(redBadRecords.size(),
+        counters.findCounter(TaskCounter.REDUCE_SKIPPED_GROUPS).getCounter());
+    assertEquals(redRecs,
+        counters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).getCounter());
+    assertEquals(redRecs,
+        counters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getCounter());
+    assertEquals(redRecs,
+        counters.findCounter(TaskCounter.REDUCE_OUTPUT_RECORDS).getCounter());
     
     //validate skipped records
     Path skipDir = SkipBadRecords.getSkipOutputPath(conf);

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestDebugScript.java
Tue Mar  8 05:57:12 2011
@@ -151,8 +151,8 @@ public class TestDebugScript {
     File output = TaskLog.getRealTaskLogFileLocation(taskId, false,
         TaskLog.LogName.DEBUGOUT);
     // Check the presence of the output file if the script is to be run.
-    assertTrue("Output file does not exists. DebugScript has not been run",
-          output.exists());
+    assertTrue("Output file does not exist. DebugScript has not been run.",
+          output != null && output.exists());
     // slurp the output from file, which is one line
     BufferedReader reader = new BufferedReader(new FileReader(output));
     String out = reader.readLine();
@@ -183,6 +183,7 @@ public class TestDebugScript {
       throws IOException {
     conf.setMapDebugScript(SCRIPT_FILE);
     conf.setMaxMapAttempts(0);
+    conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
     conf.set("mapred.committer.job.setup.cleanup.needed", "false");
     RunningJob rJob = UtilsForTests.runJobFail(conf, inputPath, outputDir);
     return rJob.getID();

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestIsolationRunner.java
Tue Mar  8 05:57:12 2011
@@ -131,6 +131,7 @@ public class TestIsolationRunner extends
       conf.setKeepTaskFilesPattern(".*");
       conf.set(SIDE_EFFECT_DIR_PROPERTY, TEST_ROOT_DIR +
           "/isolationrunnerjob/sideeffect");
+      conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
       // Delete previous runs' data.
       deleteSideEffectFiles(conf);
       JobID jobId = runJobNormally(conf);

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
Tue Mar  8 05:57:12 2011
@@ -98,8 +98,10 @@ public class TestJobCleanup extends Test
       System.err.println("---- HERE ----");
       JobConf conf = context.getJobConf();
       Path outputPath = FileOutputFormat.getOutputPath(conf);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup CommitterWithCustomDeprecatedCleanup (FileOutputCommitter) cleanupJob():
creating " + outputPath + "/" + CUSTOM_CLEANUP_FILE_NAME);
       FileSystem fs = outputPath.getFileSystem(conf);
       fs.create(new Path(outputPath, CUSTOM_CLEANUP_FILE_NAME)).close();
+//try { Thread.sleep(20000);  /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException
ie) { }
     }
   }
   
@@ -116,11 +118,14 @@ public class TestJobCleanup extends Test
       String fileName = (state == JobStatus.FAILED) 
                         ? TestJobCleanup.ABORT_FAILED_FILE_NAME 
                         : TestJobCleanup.ABORT_KILLED_FILE_NAME;
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup CommitterWithCustomAbort (FileOutputCommitter) abortJob(): creating
" + outputPath + "/" + fileName);
       fs.create(new Path(outputPath, fileName)).close();
+//try { Thread.sleep(20000);  /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException
ie) { }
     }
   }
   
   private Path getNewOutputDir() {
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup getNewOutputDir(): creating " + TEST_ROOT_DIR + "/output-" + outDirs);
     return new Path(TEST_ROOT_DIR, "output-" + outDirs++);
   }
   
@@ -150,14 +155,17 @@ public class TestJobCleanup extends Test
     JobClient jobClient = new JobClient(jc);
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testSuccessfulJob(): JobID = " + id);
     job.waitForCompletion();
     
     Path testFile = new Path(outDir, filename);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testSuccessfulJob(): checking for existence of done file: " + testFile);
     assertTrue("Done file missing for job " + id, fileSys.exists(testFile));
     
     // check if the files from the missing set exists
     for (String ex : exclude) {
       Path file = new Path(outDir, ex);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testSuccessfulJob(): checking for nonexistence of 'missing' file: "
+ file);
       assertFalse("File " + file + " should not be present for successful job " 
                   + id, fileSys.exists(file));
     }
@@ -178,10 +186,12 @@ public class TestJobCleanup extends Test
     JobClient jobClient = new JobClient(jc);
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testFailedJob(): JobID = " + id);
     job.waitForCompletion();
     
     if (fileName != null) {
       Path testFile = new Path(outDir, fileName);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testFailedJob(): checking for existence of done file: " + testFile);
       assertTrue("File " + testFile + " missing for failed job " + id, 
                  fileSys.exists(testFile));
     }
@@ -189,6 +199,7 @@ public class TestJobCleanup extends Test
     // check if the files from the missing set exists
     for (String ex : exclude) {
       Path file = new Path(outDir, ex);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testFailedJob(): checking for nonexistence of 'missing' file: " + file);
       assertFalse("File " + file + " should not be present for failed job "
                   + id, fileSys.exists(file));
     }
@@ -208,12 +219,14 @@ public class TestJobCleanup extends Test
     JobClient jobClient = new JobClient(jc);
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testKilledJob(): JobID = " + id);
     JobInProgress jip = 
       mr.getJobTrackerRunner().getJobTracker().getJob(job.getID());
     
     // wait for the map to be launched
     while (true) {
-      if (jip.runningMaps() == 1) {
+      if (jip.runningMaps() == 1
+          || (jip.isUber() && jip.runningReduces() == 1)) {
         break;
       }
       UtilsForTests.waitFor(100);
@@ -225,6 +238,7 @@ public class TestJobCleanup extends Test
     
     if (fileName != null) {
       Path testFile = new Path(outDir, fileName);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testKilledJob(): checking for existence of done file: " + testFile);
       assertTrue("File " + testFile + " missing for job " + id, 
                  fileSys.exists(testFile));
     }
@@ -232,6 +246,7 @@ public class TestJobCleanup extends Test
     // check if the files from the missing set exists
     for (String ex : exclude) {
       Path file = new Path(outDir, ex);
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testKilledJob(): checking for nonexistence of 'missing' file: " + file);
       assertFalse("File " + file + " should not be present for killed job "
                   + id, fileSys.exists(file));
     }
@@ -266,17 +281,20 @@ public class TestJobCleanup extends Test
    */
   public void testCustomAbort() throws IOException {
     // check with a successful job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomAbort(): calling testSuccessfulJob(" + FileOutputCommitter.SUCCEEDED_FILE_NAME
+ ", CommitterWithCustomAbort.class, ...)");
     testSuccessfulJob(FileOutputCommitter.SUCCEEDED_FILE_NAME, 
                       CommitterWithCustomAbort.class,
                       new String[] {ABORT_FAILED_FILE_NAME, 
                                     ABORT_KILLED_FILE_NAME});
     
     // check with a failed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomAbort(): calling testFailedJob(" + ABORT_FAILED_FILE_NAME
+ ", CommitterWithCustomAbort.class, ...)");
     testFailedJob(ABORT_FAILED_FILE_NAME, CommitterWithCustomAbort.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
                                 ABORT_KILLED_FILE_NAME});
     
     // check with a killed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomAbort(): calling testKilledJob(" + ABORT_KILLED_FILE_NAME
+ ", CommitterWithCustomAbort.class, ...)");
     testKilledJob(ABORT_KILLED_FILE_NAME, CommitterWithCustomAbort.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
                                 ABORT_FAILED_FILE_NAME});
@@ -289,16 +307,19 @@ public class TestJobCleanup extends Test
    */
   public void testCustomCleanup() throws IOException {
     // check with a successful job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomCleanup(): calling testSuccessfulJob(" + CUSTOM_CLEANUP_FILE_NAME
+ ", CommitterWithCustomDeprecatedCleanup.class, ...)");
     testSuccessfulJob(CUSTOM_CLEANUP_FILE_NAME, 
                       CommitterWithCustomDeprecatedCleanup.class,
                       new String[] {});
     
     // check with a failed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomCleanup(): calling testFailedJob(" + CUSTOM_CLEANUP_FILE_NAME
+ ", CommitterWithCustomDeprecatedCleanup.class, ...)");
     testFailedJob(CUSTOM_CLEANUP_FILE_NAME, 
                   CommitterWithCustomDeprecatedCleanup.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});
     
     // check with a killed job
+System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomCleanup(): calling testKilledJob(" + TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME
+ ", CommitterWithCustomDeprecatedCleanup.class, ...)");
     testKilledJob(TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME, 
                   CommitterWithCustomDeprecatedCleanup.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});

Modified: hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp?rev=1079218&r1=1079217&r2=1079218&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/webapps/job/jobdetails.jsp Tue Mar  8 05:57:12
2011
@@ -365,7 +365,7 @@
               "<th>Killed</th>" +
               "<th><a href=\"jobfailures.jsp?jobid=" + jobId + 
               "\">Failed/Killed<br>Task Attempts</a></th></tr>\n");
-    if (job.getUberMode()) {
+    if (job.isUber()) {
       /* placeholder until true task- and attempt-level uber info available */
       printTaskSummary(out, jobId, "uber", status.reduceProgress(),
                        job.getTasks(TaskType.REDUCE));
@@ -452,7 +452,7 @@ if("off".equals(session.getAttribute("ma
 <%} }%>
 
 <%if (job.getTasks(TaskType.REDUCE).length > 0) { %>
-<%if (job.getUberMode()) { %>
+<%if (job.isUber()) { %>
 <hr>UberTask Completion Graph -
 <%} else { %>
 <hr>Reduce Completion Graph -



Mime
View raw message