hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject svn commit: r771570 - in /hadoop/core/trunk: CHANGES.txt src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
Date Tue, 05 May 2009 05:59:00 GMT
Author: matei
Date: Tue May  5 05:59:00 2009
New Revision: 771570

URL: http://svn.apache.org/viewvc?rev=771570&view=rev
Log:
HADOOP-5745. Allow setting the default value of maxRunningJobs for all
pools. Contributed by Dhruba Borthakur.



Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
    hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771570&r1=771569&r2=771570&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May  5 05:59:00 2009
@@ -110,6 +110,9 @@
     HADOOP-5266. Adds the capability to do mark/reset of the reduce values 
     iterator in the Context object API. (Jothi Padmanabhan via ddas)
 
+    HADOOP-5745. Allow setting the default value of maxRunningJobs for all
+    pools. (dhruba via matei)
+
   IMPROVEMENTS
 
     HADOOP-4565. Added CombineFileInputFormat to use data locality information

Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java?rev=771570&r1=771569&r2=771570&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
(original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/java/org/apache/hadoop/mapred/PoolManager.java
Tue May  5 05:59:00 2009
@@ -70,6 +70,7 @@
   private Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
   private Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
   private int userMaxJobsDefault = Integer.MAX_VALUE;
+  private int poolMaxJobsDefault = Integer.MAX_VALUE;
 
   private String allocFile; // Path to XML file containing allocations
   private String poolNameProperty; // Jobconf property to use for determining a
@@ -168,6 +169,7 @@
     Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
     Map<String, Double> poolWeights = new HashMap<String, Double>();
     int userMaxJobsDefault = Integer.MAX_VALUE;
+    int poolMaxJobsDefault = Integer.MAX_VALUE;
     
     // Remember all pool names so we can display them on web UI, etc.
     List<String> poolNamesInAllocFile = new ArrayList<String>();
@@ -233,6 +235,10 @@
         String text = ((Text)element.getFirstChild()).getData().trim();
         int val = Integer.parseInt(text);
         userMaxJobsDefault = val;
+      } else if ("poolMaxJobsDefault".equals(element.getTagName())) {
+        String text = ((Text)element.getFirstChild()).getData().trim();
+        int val = Integer.parseInt(text);
+        poolMaxJobsDefault = val;
       } else {
         LOG.warn("Bad element in allocations file: " + element.getTagName());
       }
@@ -246,6 +252,7 @@
       this.poolMaxJobs = poolMaxJobs;
       this.userMaxJobs = userMaxJobs;
       this.userMaxJobsDefault = userMaxJobsDefault;
+      this.poolMaxJobsDefault = poolMaxJobsDefault;
       this.poolWeights = poolWeights;
       for (String name: poolNamesInAllocFile) {
         getPool(name);
@@ -328,7 +335,7 @@
     if (poolMaxJobs.containsKey(pool)) {
       return poolMaxJobs.get(pool);
     } else {
-      return Integer.MAX_VALUE;
+      return poolMaxJobsDefault;
     }
   }
 

Modified: hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java?rev=771570&r1=771569&r2=771570&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
(original)
+++ hadoop/core/trunk/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
Tue May  5 05:59:00 2009
@@ -316,6 +316,8 @@
     out.println("<pool name=\"poolD\">");
     out.println("<maxRunningJobs>3</maxRunningJobs>");
     out.println("</pool>");
+    // Set default limit of jobs per pool to 15
+    out.println("<poolMaxJobsDefault>15</poolMaxJobsDefault>");
     // Set default limit of jobs per user to 5
     out.println("<userMaxJobsDefault>5</userMaxJobsDefault>");
     // Give user1 a limit of 10 jobs
@@ -341,7 +343,9 @@
     assertEquals(0, poolManager.getAllocation("poolC", TaskType.REDUCE));
     assertEquals(0, poolManager.getAllocation("poolD", TaskType.MAP));
     assertEquals(0, poolManager.getAllocation("poolD", TaskType.REDUCE));
-    assertEquals(Integer.MAX_VALUE, poolManager.getPoolMaxJobs("poolA"));
+    assertEquals(15, poolManager.getPoolMaxJobs("poolA"));
+    assertEquals(15, poolManager.getPoolMaxJobs("poolB"));
+    assertEquals(15, poolManager.getPoolMaxJobs("poolC"));
     assertEquals(3, poolManager.getPoolMaxJobs("poolD"));
     assertEquals(10, poolManager.getUserMaxJobs("user1"));
     assertEquals(5, poolManager.getUserMaxJobs("user2"));



Mime
View raw message