hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r937732 - in /hadoop/mapreduce/trunk: CHANGES.txt src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java
Date Sun, 25 Apr 2010 01:22:10 GMT
Author: cdouglas
Date: Sun Apr 25 01:22:10 2010
New Revision: 937732

URL: http://svn.apache.org/viewvc?rev=937732&view=rev
Log:
MAPREDUCE-1494. Ensure TestJobDirCleanup verifies the correct paths.
Contributed by Amareshwari Sriramadasu

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=937732&r1=937731&r2=937732&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Sun Apr 25 01:22:10 2010
@@ -556,6 +556,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1695. Include capacity scheduler in findbugs and javadoc-dev
     targets and also fix existing warnings. (Hong Tang via yhemanth)
 
+    MAPREDUCE-1494. Ensure TestJobDirCleanup verifies the correct paths.
+    (Amareshwari Sriramadasu via cdouglas)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java?rev=937732&r1=937731&r2=937732&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobDirCleanup.java
Sun Apr 25 01:22:10 2010
@@ -22,14 +22,14 @@ import java.io.IOException;
 
 import junit.framework.TestCase;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.SleepJob;
-import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.security.UserGroupInformation;
 
 public class TestJobDirCleanup extends TestCase {
   //The testcase brings up a cluster with many trackers, and
@@ -37,13 +37,15 @@ public class TestJobDirCleanup extends T
   //to see whether the job directories are cleaned up at the
   //end of the job (indirectly testing whether all tasktrackers
   //got a KillJobAction).
-  private static final Log LOG =
-    LogFactory.getLog(TestEmptyJob.class.getName());
-  private void runSleepJob(JobConf conf) throws Exception {
-    String[] args = { "-m", "1", "-r", "10", "-mt", "1000", "-rt", "10000" };
-    ToolRunner.run(conf, new SleepJob(), args);
+  private JobID runSleepJob(JobConf conf) throws Exception {
+    SleepJob sleep = new SleepJob();
+    sleep.setConf(conf);
+    Job job = sleep.createJob(1, 10, 1000, 1, 10000, 1);
+    job.waitForCompletion(true);
+    return job.getID();
   }
-  public void testJobDirCleanup() throws IOException {
+
+  public void testJobDirCleanup() throws Exception {
     String namenode = null;
     MiniDFSCluster dfs = null;
     MiniMRCluster mr = null;
@@ -61,12 +63,13 @@ public class TestJobDirCleanup extends T
       // make cleanup inline sothat validation of existence of these directories
       // can be done
       mr.setInlineCleanupThreads();
-      final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
+
+      // run the sleep job
       JobConf jobConf = mr.createJobConf();
-      runSleepJob(jobConf);
-      verifyJobDirCleanup(mr, taskTrackers);
-    } catch (Exception ee){
-      assertTrue(false);
+      JobID jobid = runSleepJob(jobConf);
+      
+      // verify the job directories are cleaned up.
+      verifyJobDirCleanup(mr, taskTrackers, jobid);
     } finally {
       if (fileSys != null) { fileSys.close(); }
       if (dfs != null) { dfs.shutdown(); }
@@ -74,15 +77,35 @@ public class TestJobDirCleanup extends T
     }
   }
 
-  static void verifyJobDirCleanup(MiniMRCluster mr, int numTT)
-  throws IOException {
+  static void verifyJobDirCleanup(MiniMRCluster mr, int numTT, JobID jobid)
+      throws IOException {
+    // wait till killJobAction is sent to all trackers.
+    // this loops waits atmost for 10 seconds
+    boolean sent = true;
+    for (int i = 0; i < 100; i++) {
+      sent = true;
+      for (int j = 0; j < numTT; j++) {
+        if (mr.getTaskTrackerRunner(j).getTaskTracker().getRunningJob(
+            org.apache.hadoop.mapred.JobID.downgrade(jobid)) != null) {
+          sent = false;
+          break;
+        }
+      }
+      if (!sent) {
+        UtilsForTests.waitFor(100);
+      } else {
+        break;
+      }
+    }
+    
+    assertTrue("KillJobAction not sent for all trackers", sent);
+    String user = UserGroupInformation.getCurrentUser().getShortUserName();
+    String jobDirStr = TaskTracker.getLocalJobDir(user, jobid.toString());
     for(int i=0; i < numTT; ++i) {
-      String jobDirStr = mr.getTaskTrackerLocalDir(i)+
-      "/taskTracker/jobcache";
-      File jobDir = new File(jobDirStr);
-      String[] contents = jobDir.list();
-      assertTrue("Contents of " + jobDir + " not cleanup.",
-                 (contents == null || contents.length == 0));
+      for (String localDir : mr.getTaskTrackerLocalDirs(i)) {
+        File jobDir = new File(localDir, jobDirStr);
+        assertFalse(jobDir + " is not cleaned up.", jobDir.exists());
+      }
     }
   }
 }

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java?rev=937732&r1=937731&r2=937732&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestSeveral.java Sun Apr
25 01:22:10 2010
@@ -367,7 +367,7 @@ public class TestSeveral extends TestCas
     assertFalse("Missing event notification on failing a running job", 
         myListener.contains(jobId));
 
-    TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT);
+    TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT, job.getID());
   }
 
   /**
@@ -426,7 +426,7 @@ public class TestSeveral extends TestCas
     assertFalse("Missing event notification on killing a running job", 
         myListener.contains(job.getID()));
 
-    TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT);
+    TestJobDirCleanup.verifyJobDirCleanup(mrCluster, numTT, job.getID());
   }
 
 }



Mime
View raw message