hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r981745 - in /hadoop/mapreduce/trunk: ./ src/test/mapred/org/apache/hadoop/tools/rumen/ src/test/tools/data/rumen/small-trace-test/ src/tools/org/apache/hadoop/tools/rumen/
Date Tue, 03 Aug 2010 05:46:20 GMT
Author: amareshwari
Date: Tue Aug  3 05:46:19 2010
New Revision: 981745

URL: http://svn.apache.org/viewvc?rev=981745&view=rev
Log:
MAPREDUCE-1982. Fixes Rumen's TraceBuilder to extract job name from either of configuration
properties mapreduce.job.name and mapred.job.name. Contributed by Ravi Gummadi

Added:
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Aug  3 05:46:19 2010
@@ -216,6 +216,10 @@ Trunk (unreleased changes)
     the owner of the localized file in the mapping from cache URIs to
     CacheStatus objects. (ddas)
 
+    MAPREDUCE-1982. Fixes Rumen's TraceBuilder to extract job name from either 
+    of configuration properties "mapreduce.job.name" and "mapred.job.name".
+    (Ravi Gummadi via amareshwari)
+
 Release 0.21.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
Tue Aug  3 05:46:19 2010
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.MapReduceTestUtil;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskID;
@@ -380,15 +381,34 @@ public class TestRumenJobTraces {
       lfs.delete(tempDir, true);
     }
   }
-  
+
   @Test
   public void testJobConfigurationParser() throws Exception {
-    String[] list1 =
-        { "mapred.job.queue.name", "mapreduce.job.name",
-            "mapred.child.java.opts" };
 
-    String[] list2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
+    // Validate parser with old mapred config properties from
+    // sample-conf-file.xml
+    String[] oldProps1 = { "mapred.job.queue.name", "mapred.job.name",
+        "mapred.child.java.opts" };
+
+    String[] oldProps2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
+
+    validateJobConfParser(oldProps1, oldProps2, "sample-conf.file.xml", false);
+
+    // Validate parser with new mapreduce config properties from
+    // sample-conf-file.new.xml
+    String[] newProps1 = { MRJobConfig.QUEUE_NAME, MRJobConfig.JOB_NAME,
+        MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
+
+    String[] newProps2 = { MRJobConfig.QUEUE_NAME,
+        MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
+
+    validateJobConfParser(newProps1, newProps2,
+        "sample-conf.file.new.xml", true);
+  }
 
+  private void validateJobConfParser(String[] list1, String[] list2,
+      String confFile, boolean newConfig)
+      throws Exception {
     List<String> interested1 = new ArrayList<String>();
     for (String interested : list1) {
       interested1.add(interested);
@@ -412,7 +432,7 @@ public class TestRumenJobTraces {
 
     final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
 
-    final Path inputPath = new Path(rootInputPath, "sample-conf.file.xml");
+    final Path inputPath = new Path(rootInputPath, confFile);
 
     InputStream inputConfStream =
         new PossiblyDecompressedInputStream(inputPath, conf);
@@ -424,28 +444,82 @@ public class TestRumenJobTraces {
       inputConfStream = new PossiblyDecompressedInputStream(inputPath, conf);
       Properties props2 = jcp2.parse(inputConfStream);
 
-      assertEquals("testJobConfigurationParser: wrong number of properties", 3,
-          props1.size());
-      assertEquals("testJobConfigurationParser: wrong number of properties", 2,
-          props2.size());
-
-      assertEquals("prop test 1", "TheQueue", props1
-          .get("mapred.job.queue.name"));
-      assertEquals("prop test 2", "job_0001", props1.get("mapreduce.job.name"));
-      assertEquals("prop test 3",
-          "-server -Xmx640m -Djava.net.preferIPv4Stack=true", props1
-              .get("mapred.child.java.opts"));
-      assertEquals("prop test 4", "TheQueue", props2
-          .get("mapred.job.queue.name"));
-      assertEquals("prop test 5",
-          "-server -Xmx640m -Djava.net.preferIPv4Stack=true", props2
-              .get("mapred.child.java.opts"));
+      assertEquals("testJobConfigurationParser: wrong number of properties",
+          list1.length, props1.size());
+      assertEquals("testJobConfigurationParser: wrong number of properties",
+          list2.length, props2.size());
+
+      // Make sure that parser puts the interested properties into props1 and
+      // props2 as defined by list1 and list2.
+      String oldOrNew = newConfig ? "New" : "Old";
+      assertEquals(oldOrNew + " config property for job queue name is not "
+          + " extracted properly.", "TheQueue",
+          JobBuilder.extract(props1, JobConfPropertyNames.QUEUE_NAMES
+          .getCandidates(), null));
+      assertEquals(oldOrNew + " config property for job name is not "
+          + " extracted properly.", "MyMRJob",
+          JobBuilder.extract(props1, JobConfPropertyNames.JOB_NAMES
+          .getCandidates(), null));
+
+      assertEquals(oldOrNew + " config property for job queue name is not "
+          + " extracted properly.", "TheQueue",
+          JobBuilder.extract(props2, JobConfPropertyNames.QUEUE_NAMES
+          .getCandidates(), null));
+      
+      // This config property is not interested for props2. So props should not
+      // contain this.
+      assertNull("Uninterested " + oldOrNew + " config property for job name "
+          + " is extracted.",
+          JobBuilder.extract(props2, JobConfPropertyNames.JOB_NAMES
+          .getCandidates(), null));
+
+      validateChildJavaOpts(newConfig, props1);
+      validateChildJavaOpts(newConfig, props2);
 
     } finally {
       inputConfStream.close();
     }
   }
 
+  // Validate child java opts in properties.
+  // newConfigProperties: boolean that specifies if the config properties to be
+  // validated are new OR old.
+  private void validateChildJavaOpts(boolean newConfigProperties,
+      Properties props) {
+    if (newConfigProperties) {
+      assertEquals("New config property " + MRJobConfig.MAP_JAVA_OPTS
+          + " is not extracted properly.",
+          "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+          JobBuilder.extract(props, JobConfPropertyNames.MAP_JAVA_OPTS_S
+          .getCandidates(), null));
+      assertEquals("New config property " + MRJobConfig.REDUCE_JAVA_OPTS
+          + " is not extracted properly.",
+          "-server -Xmx650m -Djava.net.preferIPv4Stack=true",
+          JobBuilder.extract(props, JobConfPropertyNames.REDUCE_JAVA_OPTS_S
+          .getCandidates(), null));
+    }
+    else {
+      // if old property mapred.child.java.opts is set, then extraction of all
+      // the following 3 properties should give that value.
+      assertEquals("mapred.child.java.opts is not extracted properly.",
+          "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+          JobBuilder.extract(props, JobConfPropertyNames.TASK_JAVA_OPTS_S
+          .getCandidates(), null));
+      assertEquals("New config property " + MRJobConfig.MAP_JAVA_OPTS
+          + " is not extracted properly when the old config property "
+          + "mapred.child.java.opts is set.",
+          "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+          JobBuilder.extract(props, JobConfPropertyNames.MAP_JAVA_OPTS_S
+          .getCandidates(), null));
+      assertEquals("New config property " + MRJobConfig.REDUCE_JAVA_OPTS
+              + " is not extracted properly when the old config property "
+              + "mapred.child.java.opts is set.",
+          "-server -Xmx640m -Djava.net.preferIPv4Stack=true",
+          JobBuilder.extract(props, JobConfPropertyNames.REDUCE_JAVA_OPTS_S
+          .getCandidates(), null));
+    }
+  }
+
   @Test
   public void testTopologyBuilder() throws Exception {
     final TopologyBuilder subject = new TopologyBuilder();

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml?rev=981745&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml
(added)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.new.xml
Tue Aug  3 05:46:19 2010
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+<!--
+New mapreduce config properties
+-->
+   <property>
+      <name>mapreduce.job.queuename</name><value>TheQueue</value>
+   </property>
+   <property>
+      <name>mapreduce.job.name</name><value>MyMRJob</value>
+   </property>
+   <property>
+      <name>maproduce.uninteresting.property</name><value>abcdef</value>
+   </property>
+   <property><name>mapreduce.map.java.opts</name><value>-server -Xmx640m
-Djava.net.preferIPv4Stack=true</value></property>
+   <property><name>mapreduce.reduce.java.opts</name><value>-server
-Xmx650m -Djava.net.preferIPv4Stack=true</value></property>
+</configuration>

Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml
(original)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-conf.file.xml
Tue Aug  3 05:46:19 2010
@@ -19,11 +19,14 @@
  */
 -->
 <configuration>
+<!--
+Old mapred config properties
+-->
    <property>
       <name>mapred.job.queue.name</name><value>TheQueue</value>
    </property>
    <property>
-      <name>mapreduce.job.name</name><value>job_0001</value>
+      <name>mapred.job.name</name><value>MyMRJob</value>
    </property>
    <property>
       <name>maproduce.uninteresting.property</name><value>abcdef</value>

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Tue Aug
 3 05:46:19 2010
@@ -164,7 +164,7 @@ public class JobBuilder {
           "JobBuilder.process(HistoryEvent): unknown event type");
   }
 
-  private String extract(Properties conf, String[] names, String defaultValue) {
+  static String extract(Properties conf, String[] names, String defaultValue) {
     for (String name : names) {
       String result = conf.getProperty(name);
 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java?rev=981745&r1=981744&r2=981745&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfPropertyNames.java
Tue Aug  3 05:46:19 2010
@@ -20,8 +20,9 @@ package org.apache.hadoop.tools.rumen;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 
 public enum JobConfPropertyNames {
-  QUEUE_NAMES("mapred.job.queue.name", MRJobConfig.QUEUE_NAME), JOB_NAMES(
-      MRJobConfig.JOB_NAME), TASK_JAVA_OPTS_S("mapred.child.java.opts"),
+  QUEUE_NAMES("mapred.job.queue.name", MRJobConfig.QUEUE_NAME),
+  JOB_NAMES("mapred.job.name", MRJobConfig.JOB_NAME),
+  TASK_JAVA_OPTS_S("mapred.child.java.opts"),
   MAP_JAVA_OPTS_S("mapred.child.java.opts", MRJobConfig.MAP_JAVA_OPTS),
   REDUCE_JAVA_OPTS_S("mapred.child.java.opts", MRJobConfig.REDUCE_JAVA_OPTS);
 



Mime
View raw message