hadoop-mapreduce-commits mailing list archives

Author: schen
Date: Fri Oct 29 01:35:09 2010
New Revision: 1028584

URL: http://svn.apache.org/viewvc?rev=1028584&view=rev
Log:
MAPREDUCE-1818. RaidNode can specify scheduling parameters. (Ramkumar Vadali
via schen)

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
    hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1028584&r1=1028583&r2=1028584&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Oct 29 01:35:09 2010
@@ -149,6 +149,9 @@ Trunk (unreleased changes)
     HADOOP-7007. Update the hudson-test-patch ant target to work with the
     latest test-patch.sh script (gkesavan)
 
+    MAPREDUCE-1818. RaidNode can specify scheduling parameters. (Ramkumar
+    Vadali via schen)
+
   OPTIMIZATIONS
 
     MAPREDUCE-1354. Enhancements to JobTracker for better performance and

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java?rev=1028584&r1=1028583&r2=1028584&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
Fri Oct 29 01:35:09 2010
@@ -249,16 +249,35 @@ public class DistRaid {
   }
 
   /**
+   * Set options specified in raid.scheduleroption.
+   * The string should be formatted as key:value[,key:value]*
+   */
+  static void setSchedulerOption(Configuration conf) {
+    String schedulerOption = conf.get("raid.scheduleroption");
+    if (schedulerOption != null) {
+      // Parse the scheduler option to get key:value pairs.
+      String[] keyValues = schedulerOption.trim().split(",");
+      for (String keyValue: keyValues) {
+        String[] fields = keyValue.trim().split(":");
+        String key = fields[0].trim();
+        String value = fields[1].trim();
+        conf.set(key, value);
+      }
+    }
+  }
+
+  /**
    * create new job conf based on configuration passed.
    * 
    * @param conf
    * @return
    */
-  private static JobConf createJobConf(Configuration conf) {
+  static JobConf createJobConf(Configuration conf) {
     JobConf jobconf = new JobConf(conf, DistRaid.class);
     jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
     jobconf.setJobName(jobName);
     jobconf.setMapSpeculativeExecution(false);
+    setSchedulerOption(jobconf);
 
     jobconf.setJarByClass(DistRaid.class);
     jobconf.setInputFormat(DistRaidInputFormat.class);

Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=1028584&r1=1028583&r2=1028584&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Fri Oct 29 01:35:09 2010
@@ -655,4 +655,13 @@ public class TestRaidNode extends TestCa
     }
     LOG.info("Test testSuspendTraversal completed.");
   }
+
+  public void testSchedulerOption() {
+    Configuration conf = new Configuration();
+    conf.set("raid.scheduleroption",
+      "mapred.fairscheduler.pool:dummy,foo:bar");
+    org.apache.hadoop.mapred.JobConf jobConf = DistRaid.createJobConf(conf);
+    assertEquals("dummy", jobConf.get("mapred.fairscheduler.pool"));
+    assertEquals("bar", jobConf.get("foo"));
+  }
 }