hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079219 - in /hadoop/mapreduce/branches/yahoo-merge/src: java/org/apache/hadoop/mapred/JobInProgress.java test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
Date Tue, 08 Mar 2011 05:57:19 GMT
Author: omalley
Date: Tue Mar  8 05:57:18 2011
New Revision: 1079219

URL: http://svn.apache.org/viewvc?rev=1079219&view=rev
Log:
commit ce9420a251cac5a2e3273818f967680b1dded3cc
Author: Greg Roelofs <roelofs@yahoo-inc.com>
Date:   Tue Dec 14 19:12:31 2010 -0800

    Clarify FIXMEs; remove debugging noise.

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1079219&r1=1079218&r2=1079219&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/JobInProgress.java
Tue Mar  8 05:57:18 2011
@@ -776,9 +776,10 @@ public class JobInProgress {
     return maps.length == 0 && reduces.length == 0 && !jobSetupCleanupNeeded;
   }
   
-//GRR FIXME?  who calls this, and what should it return for ubertasks?
+  // called only by JT's initJob() (and test-related clones), which calls it
+  // only after calling initTasks(), which may modify jobSetupCleanupNeeded:
   synchronized boolean isSetupCleanupRequired() {
-   return jobSetupCleanupNeeded;
+    return jobSetupCleanupNeeded;
   }
 
   // Should be called once the init is done. This will complete the job 
@@ -891,7 +892,6 @@ public class JobInProgress {
       // case), so henceforth we pretend that setup and cleanup aren't needed
       // --unless/until job fails or is killed, in which case a separate
       // cleanup task will be triggered
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress initSetupCleanupTasks(): forcing jobSetupCleanupNeeded to false");
       jobSetupCleanupNeeded = false;
       return;
     }
@@ -1498,19 +1498,16 @@ System.out.println("GRR DEBUG (" + Strin
                                              int numUniqueHosts,
                                              boolean isMapSlot
                                             ) throws IOException {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): starting");
     if(!tasksInited.get() || !jobSetupCleanupNeeded) {
       return null;
     }
     
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): about to enter locked section");
     synchronized(this) {
       if (!canLaunchJobCleanupTask()) {
         return null;
       }
       
       String taskTracker = tts.getTrackerName();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): taskTracker = " + taskTracker);
       // Update the last-known clusterSize
       this.clusterSize = clusterSize;
       if (!shouldRunOnTaskTracker(taskTracker)) {
@@ -1525,14 +1522,12 @@ System.out.println("GRR DEBUG (" + Strin
       }
       TaskInProgress tip = findTaskFromList(cleanupTaskList,
                              tts, numUniqueHosts, false);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): found cleanup TIP " + tip);
       if (tip == null) {
         return null;
       }
       
       // Now launch the cleanupTask
       Task result = tip.getTaskToRun(tts.getTrackerName());
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobCleanupTask(): launching cleanup Task " + result);
       if (result != null) {
         addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
         if (jobFailed) {
@@ -1594,13 +1589,11 @@ System.out.println("GRR DEBUG (" + Strin
                                              int numUniqueHosts,
                                              boolean isMapSlot
                                             ) throws IOException {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobSetupTask(): starting");
-    if(!tasksInited.get() || !jobSetupCleanupNeeded) {
+    // uberMode condition should be redundant, but make sure anyway:
+    if (!tasksInited.get() || !jobSetupCleanupNeeded || uberMode) {
       return null;
     }
-//GRR FIXME:  need special protection here (since no setup TIPs for ubermode)?  may need
to split jobSetupCleanupNeeded into setup and cleanup halves...
-    
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobSetupTask(): about to enter locked section");
+
     synchronized(this) {
       if (!canLaunchSetupTask()) {
         return null;
@@ -1626,7 +1619,6 @@ System.out.println("GRR DEBUG (" + Strin
       
       // Now launch the setupTask
       Task result = tip.getTaskToRun(tts.getTrackerName());
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress obtainJobSetupTask(): launching setup Task " + result);
       if (result != null) {
         addRunningTaskToTIP(tip, result.getTaskID(), tts, true);
       }
@@ -3067,7 +3059,6 @@ System.out.println("GRR DEBUG (" + Strin
     if (uberMode) {
       // restore setup/cleanup status so separate cleanup task will be launched
       // (see obtainJobCleanupTask())
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  JobInProgress terminate(): restoring jobSetupCleanupNeeded to " + jobSetupCleanupNeeded);
       jobSetupCleanupNeeded = uberSetupCleanupNeeded;
     }
 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1079219&r1=1079218&r2=1079219&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJobCleanup.java
Tue Mar  8 05:57:18 2011
@@ -98,10 +98,8 @@ public class TestJobCleanup extends Test
       System.err.println("---- HERE ----");
       JobConf conf = context.getJobConf();
       Path outputPath = FileOutputFormat.getOutputPath(conf);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup CommitterWithCustomDeprecatedCleanup (FileOutputCommitter) cleanupJob():
creating " + outputPath + "/" + CUSTOM_CLEANUP_FILE_NAME);
       FileSystem fs = outputPath.getFileSystem(conf);
       fs.create(new Path(outputPath, CUSTOM_CLEANUP_FILE_NAME)).close();
-//try { Thread.sleep(20000);  /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException
ie) { }
     }
   }
   
@@ -118,14 +116,11 @@ System.out.println("GRR DEBUG (" + Strin
       String fileName = (state == JobStatus.FAILED) 
                         ? TestJobCleanup.ABORT_FAILED_FILE_NAME 
                         : TestJobCleanup.ABORT_KILLED_FILE_NAME;
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup CommitterWithCustomAbort (FileOutputCommitter) abortJob(): creating
" + outputPath + "/" + fileName);
       fs.create(new Path(outputPath, fileName)).close();
-//try { Thread.sleep(20000);  /* GRR DEBUG ONLY! (arg = milliseconds) */ } catch (InterruptedException
ie) { }
     }
   }
   
   private Path getNewOutputDir() {
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup getNewOutputDir(): creating " + TEST_ROOT_DIR + "/output-" + outDirs);
     return new Path(TEST_ROOT_DIR, "output-" + outDirs++);
   }
   
@@ -155,17 +150,14 @@ System.out.println("GRR DEBUG (" + Strin
     JobClient jobClient = new JobClient(jc);
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testSuccessfulJob(): JobID = " + id);
     job.waitForCompletion();
     
     Path testFile = new Path(outDir, filename);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testSuccessfulJob(): checking for existence of done file: " + testFile);
     assertTrue("Done file missing for job " + id, fileSys.exists(testFile));
     
     // check if the files from the missing set exists
     for (String ex : exclude) {
       Path file = new Path(outDir, ex);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testSuccessfulJob(): checking for nonexistence of 'missing' file: "
+ file);
       assertFalse("File " + file + " should not be present for successful job " 
                   + id, fileSys.exists(file));
     }
@@ -186,12 +178,10 @@ System.out.println("GRR DEBUG (" + Strin
     JobClient jobClient = new JobClient(jc);
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testFailedJob(): JobID = " + id);
     job.waitForCompletion();
     
     if (fileName != null) {
       Path testFile = new Path(outDir, fileName);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testFailedJob(): checking for existence of done file: " + testFile);
       assertTrue("File " + testFile + " missing for failed job " + id, 
                  fileSys.exists(testFile));
     }
@@ -199,7 +189,6 @@ System.out.println("GRR DEBUG (" + Strin
     // check if the files from the missing set exists
     for (String ex : exclude) {
       Path file = new Path(outDir, ex);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testFailedJob(): checking for nonexistence of 'missing' file: " + file);
       assertFalse("File " + file + " should not be present for failed job "
                   + id, fileSys.exists(file));
     }
@@ -219,7 +208,6 @@ System.out.println("GRR DEBUG (" + Strin
     JobClient jobClient = new JobClient(jc);
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testKilledJob(): JobID = " + id);
     JobInProgress jip = 
       mr.getJobTrackerRunner().getJobTracker().getJob(job.getID());
     
@@ -238,7 +226,6 @@ System.out.println("GRR DEBUG (" + Strin
     
     if (fileName != null) {
       Path testFile = new Path(outDir, fileName);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testKilledJob(): checking for existence of done file: " + testFile);
       assertTrue("File " + testFile + " missing for job " + id, 
                  fileSys.exists(testFile));
     }
@@ -246,7 +233,6 @@ System.out.println("GRR DEBUG (" + Strin
     // check if the files from the missing set exists
     for (String ex : exclude) {
       Path file = new Path(outDir, ex);
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testKilledJob(): checking for nonexistence of 'missing' file: " + file);
       assertFalse("File " + file + " should not be present for killed job "
                   + id, fileSys.exists(file));
     }
@@ -281,20 +267,17 @@ System.out.println("GRR DEBUG (" + Strin
    */
   public void testCustomAbort() throws IOException {
     // check with a successful job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomAbort(): calling testSuccessfulJob(" + FileOutputCommitter.SUCCEEDED_FILE_NAME
+ ", CommitterWithCustomAbort.class, ...)");
     testSuccessfulJob(FileOutputCommitter.SUCCEEDED_FILE_NAME, 
                       CommitterWithCustomAbort.class,
                       new String[] {ABORT_FAILED_FILE_NAME, 
                                     ABORT_KILLED_FILE_NAME});
     
     // check with a failed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomAbort(): calling testFailedJob(" + ABORT_FAILED_FILE_NAME
+ ", CommitterWithCustomAbort.class, ...)");
     testFailedJob(ABORT_FAILED_FILE_NAME, CommitterWithCustomAbort.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
                                 ABORT_KILLED_FILE_NAME});
     
     // check with a killed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomAbort(): calling testKilledJob(" + ABORT_KILLED_FILE_NAME
+ ", CommitterWithCustomAbort.class, ...)");
     testKilledJob(ABORT_KILLED_FILE_NAME, CommitterWithCustomAbort.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME,
                                 ABORT_FAILED_FILE_NAME});
@@ -307,19 +290,16 @@ System.out.println("GRR DEBUG (" + Strin
    */
   public void testCustomCleanup() throws IOException {
     // check with a successful job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomCleanup(): calling testSuccessfulJob(" + CUSTOM_CLEANUP_FILE_NAME
+ ", CommitterWithCustomDeprecatedCleanup.class, ...)");
     testSuccessfulJob(CUSTOM_CLEANUP_FILE_NAME, 
                       CommitterWithCustomDeprecatedCleanup.class,
                       new String[] {});
     
     // check with a failed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomCleanup(): calling testFailedJob(" + CUSTOM_CLEANUP_FILE_NAME
+ ", CommitterWithCustomDeprecatedCleanup.class, ...)");
     testFailedJob(CUSTOM_CLEANUP_FILE_NAME, 
                   CommitterWithCustomDeprecatedCleanup.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});
     
     // check with a killed job
-System.out.println("GRR DEBUG (" + String.format("%1$tF %1$tT,%1$tL", System.currentTimeMillis())
+ "):  TestJobCleanup testCustomCleanup(): calling testKilledJob(" + TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME
+ ", CommitterWithCustomDeprecatedCleanup.class, ...)");
     testKilledJob(TestJobCleanup.CUSTOM_CLEANUP_FILE_NAME, 
                   CommitterWithCustomDeprecatedCleanup.class, 
                   new String[] {FileOutputCommitter.SUCCEEDED_FILE_NAME});



Mime
View raw message