hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1061071 - in /hadoop/mapreduce/branches/branch-0.22: CHANGES.txt src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
Date Thu, 20 Jan 2011 00:15:02 GMT
Author: todd
Date: Thu Jan 20 00:15:01 2011
New Revision: 1061071

URL: http://svn.apache.org/viewvc?rev=1061071&view=rev
Log:
MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test. Contributed by Todd
Lipcon

Modified:
    hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
    hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java

Modified: hadoop/mapreduce/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/CHANGES.txt?rev=1061071&r1=1061070&r2=1061071&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.22/CHANGES.txt Thu Jan 20 00:15:01 2011
@@ -178,6 +178,8 @@ Release 0.22.0 - Unreleased
 
     MAPREDUCE-1878. Add MRUnit documentation. (Aaron Kimball via tomwhite)
 
+    MAPREDUCE-2180. Add coverage of fair scheduler servlet to system test (todd)
+
   OPTIMIZATIONS
 
     MAPREDUCE-1354. Enhancements to JobTracker for better performance and

Modified: hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java?rev=1061071&r1=1061070&r2=1061071&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
(original)
+++ hadoop/mapreduce/branches/branch-0.22/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairSchedulerSystem.java
Thu Jan 20 00:15:01 2011
@@ -21,23 +21,63 @@ import org.apache.hadoop.mapreduce.serve
 import org.apache.hadoop.mapreduce.SleepJob;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.conf.Configuration;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.HttpURLConnection;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.junit.Test;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
 import static org.junit.Assert.*;
 
 /**
  * System tests for the fair scheduler. These run slower than the
  * mock-based tests in TestFairScheduler but have a better chance
  * of catching synchronization bugs with the real JT.
+ *
+ * This test suite will often be run inside JCarder in order to catch
+ * deadlock bugs which have plagued the scheduler in the past - hence
+ * it is a bit of a "grab-bag" of system tests, since it's important
+ * that they all run as part of the same JVM instantiation.
  */
 public class TestFairSchedulerSystem {
-  static final int NUM_THREADS=5;
+  static final int NUM_THREADS=2;
+
+  static MiniMRCluster mr;
+  static JobConf conf;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    conf = new JobConf();
+    final int taskTrackers = 1;
+
+    // Bump up the frequency of preemption updates to test against
+    // deadlocks, etc.
+    conf.set("mapred.jobtracker.taskScheduler", FairScheduler.class.getCanonicalName());
+    conf.set("mapred.fairscheduler.update.interval", "1");
+    conf.set("mapred.fairscheduler.preemption.interval", "1");
+    conf.set("mapred.fairscheduler.preemption", "true");
+    conf.set("mapred.fairscheduler.eventlog.enabled", "true");
+    conf.set(JTConfig.JT_PERSIST_JOBSTATUS, "false");
+    mr = new MiniMRCluster(taskTrackers, "file:///", 1, null, null, conf);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    if (mr != null) {
+      mr.shutdown();
+    }
+  }
 
   private void runSleepJob(JobConf conf) throws Exception {
     String[] args = { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" };
@@ -45,44 +85,64 @@ public class TestFairSchedulerSystem {
   }
 
   /**
-   * Bump up the frequency of preemption updates to test against
-   * deadlocks, etc.
+   * Submit some concurrent sleep jobs, and visit the scheduler servlet
+   * while they're running.
    */
   @Test
-  public void testPreemptionUpdates() throws Exception {
-    MiniMRCluster mr = null;
-    try {
-      final int taskTrackers = 1;
-
-      JobConf conf = new JobConf();
-      conf.set("mapred.jobtracker.taskScheduler", FairScheduler.class.getCanonicalName());
-      conf.set("mapred.fairscheduler.update.interval", "0");
-      conf.set("mapred.fairscheduler.preemption.interval", "0");
-      conf.set("mapred.fairscheduler.preemption", "true");
-      conf.set("mapred.fairscheduler.eventlog.enabled", "true");
-      conf.set(JTConfig.JT_PERSIST_JOBSTATUS, "false");
-
-      mr = new MiniMRCluster(taskTrackers, "file:///", 1, null, null, conf);
-      final MiniMRCluster finalMR = mr;
-      ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
-      List<Future<Void>> futures = new ArrayList<Future<Void>>(NUM_THREADS);
-      for (int i = 0; i < NUM_THREADS; i++) {
-        futures.add(exec.submit(new Callable<Void>() {
-          public Void call() throws Exception {
-            JobConf jobConf = finalMR.createJobConf();
-            runSleepJob(jobConf);
-            return null;
-          }
-        }));
-      }
+  public void testFairSchedulerSystem() throws Exception {
+    ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
+    List<Future<Void>> futures = new ArrayList<Future<Void>>(NUM_THREADS);
+    for (int i = 0; i < NUM_THREADS; i++) {
+      futures.add(exec.submit(new Callable<Void>() {
+            public Void call() throws Exception {
+              JobConf jobConf = mr.createJobConf();
+              runSleepJob(jobConf);
+              return null;
+            }
+          }));
+    }
 
-      for (Future<Void> future : futures) {
-        future.get();
-      }
-    } finally {
-      if (mr != null) { mr.shutdown();
+    // Wait for the tasks to finish, and visit the scheduler servlet
+    // every few seconds while waiting.
+    for (Future<Void> future : futures) {
+      while (true) {
+        try {
+          future.get(3, TimeUnit.SECONDS);
+          break;
+        } catch (TimeoutException te) {
+          // It's OK
+        }
+        checkServlet(true);
+        checkServlet(false);
       }
     }
-  
   }
+
+  /**
+   * Check the fair scheduler servlet for good status code and smoke test
+   * for contents.
+   */
+  private void checkServlet(boolean advanced) throws Exception {
+    String jtURL = "http://localhost:" +
+      mr.getJobTrackerRunner().getJobTrackerInfoPort();
+    URL url = new URL(jtURL + "/scheduler" +
+                      (advanced ? "?advanced" : ""));
+    HttpURLConnection connection = (HttpURLConnection)url.openConnection();
+    connection.setRequestMethod("GET");
+    connection.connect();
+    assertEquals(200, connection.getResponseCode());
+
+    // Just to be sure, slurp the content and make sure it looks like the scheduler
+    BufferedReader reader = new BufferedReader(
+      new InputStreamReader(connection.getInputStream()));
+    StringBuilder sb = new StringBuilder();
+
+    String line = null;
+    while ((line = reader.readLine()) != null) {
+      sb.append(line).append('\n');
+    }
+
+    String contents = sb.toString();
+    assertTrue(contents.contains("Fair Scheduler Administration"));
+ }
 }



Mime
View raw message