Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: core-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r727660 - in /hadoop/core/trunk: CHANGES.txt
 src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
 src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java
Date: Thu, 18 Dec 2008 07:08:41 -0000
To: core-commits@hadoop.apache.org
From: yhemanth@apache.org
Message-Id: <20081218070843.9438A2388892@eris.apache.org>

Author: yhemanth
Date: Wed Dec 17 23:08:38 2008
New Revision: 727660

URL: http://svn.apache.org/viewvc?rev=727660&view=rev
Log:
HADOOP-4774. Fix default values of some capacity scheduler configuration items. Contributed by Sreekanth Ramakrishnan

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
    hadoop/core/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=727660&r1=727659&r2=727660&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Dec 17 23:08:38 2008
@@ -471,6 +471,10 @@
     lib directory and added the same to test-patch.sh. (Giridharan Kesavan via
     acmurthy)
 
+    HADOOP-4774. Fix default values of some capacity scheduler configuration
+    items which would otherwise not work on a fresh checkout.
+    (Sreekanth Ramakrishnan via yhemanth)
+
 Release 0.19.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java?rev=727660&r1=727659&r2=727660&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java (original)
+++ hadoop/core/trunk/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacitySchedulerConf.java Wed Dec 17 23:08:38 2008
@@ -71,6 +71,18 @@
   static final String UPPER_LIMIT_ON_TASK_PMEM_PROPERTY =
       "mapred.capacity-scheduler.task.limit.maxpmem";
 
+  /**
+   * The constant which defines the default initialization thread
+   * polling interval, denoted in milliseconds.
+   */
+  private static final int INITIALIZATION_THREAD_POLLING_INTERVAL = 5000;
+
+  /**
+   * The constant which defines the maximum number of worker threads to be
+   * spawned off for job initialization
+   */
+  private static final int MAX_INITIALIZATION_WORKER_THREADS = 5;
+
   private Configuration rmConf;
 
   private int defaultMaxJobsPerUsersToInitialize;
@@ -312,15 +324,19 @@
   }
 
   /**
-   * Amount of time in miliseconds which poller thread and initialization
+   * Amount of time in milliseconds which poller thread and initialization
    * thread would sleep before looking at the queued jobs.
+   * 
+   * The default value if no corresponding configuration is present is
+   * 5000 Milliseconds.
    *  
-   * @return time in miliseconds.
+   * @return time in milliseconds.
    * @throws IllegalArgumentException if time is negative or zero.
    */
   public long getSleepInterval() {
     long sleepInterval = rmConf.getLong(
-        "mapred.capacity-scheduler.init-poll-interval", -1);
+        "mapred.capacity-scheduler.init-poll-interval", 
+        INITIALIZATION_THREAD_POLLING_INTERVAL);
     
     if(sleepInterval <= 0) {
       throw new IllegalArgumentException(
@@ -342,12 +358,15 @@
    * So a given thread can have responsibility of initializing jobs from more 
    * than one queue.
    * 
+   * The default value is 5
+   * 
    * @return maximum number of threads spawned to initialize jobs in job queue
    * in parallel.
    */
   public int getMaxWorkerThreads() {
     int maxWorkerThreads = rmConf.getInt(
-        "mapred.capacity-scheduler.init-worker-threads", 0);
+        "mapred.capacity-scheduler.init-worker-threads", 
+        MAX_INITIALIZATION_WORKER_THREADS);
     if(maxWorkerThreads <= 0) {
       throw new IllegalArgumentException(
           "Invalid initializater worker thread number " + maxWorkerThreads);

Modified: hadoop/core/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java?rev=727660&r1=727659&r2=727660&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java (original)
+++ hadoop/core/trunk/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerConf.java Wed Dec 17 23:08:38 2008
@@ -323,6 +323,55 @@
     }
   }
   
+  public void testInitializationPollerProperties() 
+    throws Exception {
+    /*
+     * Test case to check properties of poller when no configuration file
+     * is present.
+     */
+    testConf = new CapacitySchedulerConf();
+    long pollingInterval = testConf.getSleepInterval();
+    int maxWorker = testConf.getMaxWorkerThreads();
+    assertTrue("Invalid polling interval ",pollingInterval > 0);
+    assertTrue("Invalid working thread pool size" , maxWorker > 0);
+    
+    //test case for custom values configured for initialization 
+    //poller.
+    openFile();
+    startConfig();
+    writeProperty("mapred.capacity-scheduler.init-worker-threads", "1");
+    writeProperty("mapred.capacity-scheduler.init-poll-interval", "1");
+    endConfig();
+    
+    testConf = new CapacitySchedulerConf(new Path(testConfFile));
+    
+    pollingInterval = testConf.getSleepInterval();
+    
+    maxWorker = testConf.getMaxWorkerThreads();
+    
+    assertEquals("Invalid polling interval ",pollingInterval ,1);
+    assertEquals("Invalid working thread pool size" , maxWorker, 1);
+    
+    //Test case for invalid values configured for initialization
+    //poller
+    openFile();
+    startConfig();
+    writeProperty("mapred.capacity-scheduler.init-worker-threads", "0");
+    writeProperty("mapred.capacity-scheduler.init-poll-interval", "0");
+    endConfig();
+    
+    testConf = new CapacitySchedulerConf(new Path(testConfFile));
+    
+    try {
+      pollingInterval = testConf.getSleepInterval();
+      fail("Polling interval configured is illegal");
+    } catch (IllegalArgumentException e) {}
+    try {
+      maxWorker = testConf.getMaxWorkerThreads();
+      fail("Max worker thread configured is illegal");
+    } catch (IllegalArgumentException e) {}
+  }
+  
   private void checkQueueProperties(
                         CapacitySchedulerConf testConf,
                         Map<String, Map<String, String>> queueDetails) {
@@ -396,15 +445,6 @@
   }
   
   
-  private void writeUserDefinedDefaultConfigurationWithoutGC() {
-    writeProperty("mapred.capacity-scheduler.default-reclaim-time-limit"
-        , "800");
-    writeProperty("mapred.capacity-scheduler.default-supports-priority"
-        , "true");
-    writeProperty("mapred.capacity-scheduler.default-minimum-user-limit-percent"
-        , "50");    
-  }
-  
   private void writeProperty(String name, String value) {
     writer.println("<property>");
     writer.println("<name> " + name + "</name>");