hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r743996 - in /hadoop/core/trunk: CHANGES.txt src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java src/test/org/apache/hadoop/mapred/UtilsForTests.java
Date Fri, 13 Feb 2009 05:38:01 GMT
Author: ddas
Date: Fri Feb 13 05:38:01 2009
New Revision: 743996

URL: http://svn.apache.org/viewvc?rev=743996&view=rev
Log:
HADOOP-4220. Changes the JobTracker restart tests so that they take much  less time. Contributed
by Amar Kamat.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Feb 13 05:38:01 2009
@@ -101,6 +101,9 @@
     HADOOP-1722. Adds support for TypedBytes and RawBytes in Streaming.
     (Klaas Bosteels via ddas)
 
+    HADOOP-4220. Changes the JobTracker restart tests so that they take much
+    less time. (Amar Kamat via ddas)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestart.java Fri Feb
13 05:38:01 2009
@@ -121,20 +121,11 @@
     mr.getJobTrackerConf().setBoolean("mapred.jobtracker.restart.recover", 
                                       false);
     
-    // Wait for a minute before submitting a job
-    UtilsForTests.waitFor(60 * 1000);
-    
     mr.startJobTracker();
     
-    // Signal the tasks
-    UtilsForTests.signalTasks(dfs, fileSys, true, getMapSignalFile(shareDir), 
-                              getReduceSignalFile(shareDir));
-    
     // Wait for the JT to be ready
     UtilsForTests.waitForJobTracker(jobClient);
     
-    UtilsForTests.waitTillDone(jobClient);
-    
     // The submitted job should not exist
     assertTrue("Submitted job was detected with recovery disabled", 
                UtilsForTests.getJobStatus(jobClient, id) == null);
@@ -143,14 +134,13 @@
   /** Tests a job on jobtracker with restart-recovery turned on.
    * Preparation :
    *    - Configure a job with
-   *       - num-maps : 50
+   *       - num-maps : 24
    *       - num-reducers : 1
    *    - Configure the cluster to run 1 reducer
    *    - Lower the history file block size and buffer
    *    
    * Wait for the job to complete 50%. Note that all the job is configured to 
-   * use {@link HalfWaitingMapper} and {@link WaitingReducer}. So job will 
-   * eventually wait on 50%
+   * use {@link HalfWaitingMapper}. So job will eventually wait on 50%
    * 
    * Make a note of the following things
    *    - Task completion events
@@ -191,7 +181,7 @@
     //     Ideally the tracker should SYNC with the new/restarted jobtracker
     
     FileSystem fileSys = dfs.getFileSystem();
-    final int numMaps = 50;
+    final int numMaps = 24;
     final int numReducers = 1;
     
     
@@ -224,7 +214,7 @@
         mr.getMapTaskCompletionEventsUpdates(0, id, numMaps)
           .getMapTaskCompletionEvents();
       if (trackerEvents.length < numMaps / 2) {
-        UtilsForTests.waitFor(1000);
+        UtilsForTests.waitFor(100);
       } else {
         break;
       }
@@ -242,9 +232,6 @@
     mr.getJobTrackerConf().setBoolean("mapred.jobtracker.restart.recover", 
                                       true);
     
-    //  Wait for a minute before submitting a job
-    UtilsForTests.waitFor(60 * 1000);
-    
     mr.startJobTracker();
     
     // Signal the map tasks
@@ -274,7 +261,7 @@
         mr.getMapTaskCompletionEventsUpdates(0, id, 2 * numMaps)
           .getMapTaskCompletionEvents();
       if (trackerEvents.length < jtEvents.length) {
-        UtilsForTests.waitFor(1000);
+        UtilsForTests.waitFor(100);
       } else {
         break;
       }
@@ -429,10 +416,11 @@
       // Make sure that jobhistory leads to a proper job restart
       // So keep the blocksize and the buffer size small
       JobConf jtConf = new JobConf();
-      jtConf.set("mapred.jobtracker.job.history.block.size", "1024");
-      jtConf.set("mapred.jobtracker.job.history.buffer.size", "1024");
+      jtConf.set("mapred.jobtracker.job.history.block.size", "64");
+      jtConf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
       jtConf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
-      jtConf.setLong("mapred.tasktracker.expiry.interval", 25 * 1000);
+      jtConf.setLong("mapred.tasktracker.expiry.interval", 5000);
+      jtConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
       jtConf.setBoolean("mapred.acls.enabled", true);
       // get the user group info
       UserGroupInformation ugi = UserGroupInformation.getCurrentUGI();

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobTrackerRestartWithLostTracker.java
Fri Feb 13 05:38:01 2009
@@ -49,7 +49,7 @@
   throws IOException {
     FileSystem fileSys = dfs.getFileSystem();
     JobConf jobConf = mr.createJobConf();
-    int numMaps = 50;
+    int numMaps = 30;
     int numReds = 1;
     String mapSignalFile = UtilsForTests.getMapSignalFile(shareDir);
     String redSignalFile = UtilsForTests.getReduceSignalFile(shareDir);
@@ -87,9 +87,6 @@
     // Kill the 2nd tasktracker
     mr.stopTaskTracker(1);
     
-    // Wait for a minute before submitting a job
-    UtilsForTests.waitFor(60 * 1000);
-    
     // Restart the jobtracker
     mr.startJobTracker();
 
@@ -104,11 +101,17 @@
     
     UtilsForTests.waitTillDone(jobClient);
 
-    // Check if the tasks on the lost tracker got re-executed
+    // Check if the tracker got lost
     assertTrue("Tracker killed while the jobtracker was down did not get lost "
                 + "upon restart", 
                 jobClient.getClusterStatus().getTaskTrackers() 
                 < mr.getNumTaskTrackers());
+    
+    //  Check if the tasks on the lost tracker got killed
+    int failedMaps = 
+      mr.getJobTrackerRunner().getJobTracker().getJob(id).failedMapTasks;
+    assertTrue("Tasks that were run on the lost tracker were not killed", 
+               failedMaps > 0);
   }
   
   public void testRestartWithLostTracker() throws IOException {
@@ -144,10 +147,10 @@
       // Make sure that jobhistory leads to a proper job restart
       // So keep the blocksize and the buffer size small
       JobConf jtConf = new JobConf();
-      jtConf.set("mapred.jobtracker.job.history.block.size", "1024");
-      jtConf.set("mapred.jobtracker.job.history.buffer.size", "1024");
+      jtConf.set("mapred.jobtracker.job.history.block.size", "512");
+      jtConf.set("mapred.jobtracker.job.history.buffer.size", "512");
       jtConf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
-      jtConf.setLong("mapred.tasktracker.expiry.interval", 25 * 1000);
+      jtConf.setLong("mapred.tasktracker.expiry.interval", 5000);
       jtConf.setInt("mapred.reduce.copy.backoff", 4);
       
       mr = new MiniMRCluster(2, namenode, 1, null, null, jtConf);

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java?rev=743996&r1=743995&r2=743996&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/UtilsForTests.java Fri Feb 13 05:38:01
2009
@@ -254,7 +254,7 @@
         }
       }
       if (shouldWait) {
-        waitFor(1000);
+        waitFor(100);
       } else {
         break;
       }



Mime
View raw message