Author: schen
Date: Fri Oct 29 01:35:09 2010
New Revision: 1028584
URL: http://svn.apache.org/viewvc?rev=1028584&view=rev
Log:
MAPREDUCE-1818. RaidNode can specify scheduling parameters. (Ramkumar Vadali
via schen)
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1028584&r1=1028583&r2=1028584&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Oct 29 01:35:09 2010
@@ -149,6 +149,9 @@ Trunk (unreleased changes)
HADOOP-7007. Update the hudson-test-patch ant target to work with the
latest test-patch.sh script (gkesavan)
+ MAPREDUCE-1818. RaidNode can specify scheduling parameters. (Ramkumar
+ Vadali via schen)
+
OPTIMIZATIONS
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java?rev=1028584&r1=1028583&r2=1028584&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/DistRaid.java
Fri Oct 29 01:35:09 2010
@@ -249,16 +249,35 @@ public class DistRaid {
}
/**
+ * Set options specified in raid.scheduleroption.
+ * The string should be formatted as key:value[,key:value]*
+ */
+ static void setSchedulerOption(Configuration conf) {
+ String schedulerOption = conf.get("raid.scheduleroption");
+ if (schedulerOption != null) {
+ // Parse the scheduler option to get key:value pairs.
+ String[] keyValues = schedulerOption.trim().split(",");
+ for (String keyValue: keyValues) {
+ String[] fields = keyValue.trim().split(":");
+ String key = fields[0].trim();
+ String value = fields[1].trim();
+ conf.set(key, value);
+ }
+ }
+ }
+
+ /**
* create new job conf based on configuration passed.
*
* @param conf
* @return
*/
- private static JobConf createJobConf(Configuration conf) {
+ static JobConf createJobConf(Configuration conf) {
JobConf jobconf = new JobConf(conf, DistRaid.class);
jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
jobconf.setJobName(jobName);
jobconf.setMapSpeculativeExecution(false);
+ setSchedulerOption(jobconf);
jobconf.setJarByClass(DistRaid.class);
jobconf.setInputFormat(DistRaidInputFormat.class);
Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=1028584&r1=1028583&r2=1028584&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Fri Oct 29 01:35:09 2010
@@ -655,4 +655,13 @@ public class TestRaidNode extends TestCa
}
LOG.info("Test testSuspendTraversal completed.");
}
+
+ public void testSchedulerOption() {
+ Configuration conf = new Configuration();
+ conf.set("raid.scheduleroption",
+ "mapred.fairscheduler.pool:dummy,foo:bar");
+ org.apache.hadoop.mapred.JobConf jobConf = DistRaid.createJobConf(conf);
+ assertEquals("dummy", jobConf.get("mapred.fairscheduler.pool"));
+ assertEquals("bar", jobConf.get("foo"));
+ }
}
|