hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ama...@apache.org
Subject svn commit: r1097325 - in /hadoop/mapreduce/trunk: ./ src/test/mapred/org/apache/hadoop/tools/rumen/ src/test/tools/data/rumen/small-trace-test/ src/tools/org/apache/hadoop/tools/rumen/
Date Thu, 28 Apr 2011 05:43:21 GMT
Author: amarrk
Date: Thu Apr 28 05:43:20 2011
New Revision: 1097325

URL: http://svn.apache.org/viewvc?rev=1097325&view=rev
Log:
Bring in more job configuration properties in to the trace file.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Apr 28 05:43:20 2011
@@ -7,6 +7,10 @@ Trunk (unreleased changes)
   NEW FEATURES
 
   IMPROVEMENTS
+
+    MAPREDUCE-2153. Bring in more job configuration properties in to the trace 
+    file. (Rajesh Balamohan via amarrk)
+
     MAPREDUCE-1461. Feature to instruct rumen-folder utility to skip jobs worth 
     of specific duration. (Rajesh Balamohan via amarrk)
 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
Thu Apr 28 05:43:20 2011
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.tools.rumen;
 
+import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -446,37 +447,12 @@ public class TestRumenJobTraces {
     String[] oldProps1 = { "mapred.job.queue.name", "mapred.job.name",
         "mapred.child.java.opts" };
 
-    String[] oldProps2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
-
-    validateJobConfParser(oldProps1, oldProps2, "sample-conf.file.xml", false);
-
-    // Validate parser with new mapreduce config properties from
-    // sample-conf-file.new.xml
-    String[] newProps1 = { MRJobConfig.QUEUE_NAME, MRJobConfig.JOB_NAME,
-        MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
-
-    String[] newProps2 = { MRJobConfig.QUEUE_NAME,
-        MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
-
-    validateJobConfParser(newProps1, newProps2,
-        "sample-conf.file.new.xml", true);
+    validateJobConfParser("sample-conf.file.xml", false);
+    validateJobConfParser("sample-conf.file.new.xml", true);
   }
 
-  private void validateJobConfParser(String[] list1, String[] list2,
-      String confFile, boolean newConfig)
+  private void validateJobConfParser(String confFile, boolean newConfig)
       throws Exception {
-    List<String> interested1 = new ArrayList<String>();
-    for (String interested : list1) {
-      interested1.add(interested);
-    }
-
-    List<String> interested2 = new ArrayList<String>();
-    for (String interested : list2) {
-      interested2.add(interested);
-    }
-
-    JobConfigurationParser jcp1 = new JobConfigurationParser(interested1);
-    JobConfigurationParser jcp2 = new JobConfigurationParser(interested2);
 
     final Configuration conf = new Configuration();
     final FileSystem lfs = FileSystem.getLocal(conf);
@@ -494,43 +470,20 @@ public class TestRumenJobTraces {
         new PossiblyDecompressedInputStream(inputPath, conf);
 
     try {
-      Properties props1 = jcp1.parse(inputConfStream);
+      Properties props = JobConfigurationParser.parse(inputConfStream);
       inputConfStream.close();
 
-      inputConfStream = new PossiblyDecompressedInputStream(inputPath, conf);
-      Properties props2 = jcp2.parse(inputConfStream);
-
-      assertEquals("testJobConfigurationParser: wrong number of properties",
-          list1.length, props1.size());
-      assertEquals("testJobConfigurationParser: wrong number of properties",
-          list2.length, props2.size());
-
-      // Make sure that parser puts the interested properties into props1 and
-      // props2 as defined by list1 and list2.
       String oldOrNew = newConfig ? "New" : "Old";
       assertEquals(oldOrNew + " config property for job queue name is not "
           + " extracted properly.", "TheQueue",
-          JobBuilder.extract(props1, JobConfPropertyNames.QUEUE_NAMES
+          JobBuilder.extract(props, JobConfPropertyNames.QUEUE_NAMES
           .getCandidates(), null));
       assertEquals(oldOrNew + " config property for job name is not "
           + " extracted properly.", "MyMRJob",
-          JobBuilder.extract(props1, JobConfPropertyNames.JOB_NAMES
-          .getCandidates(), null));
-
-      assertEquals(oldOrNew + " config property for job queue name is not "
-          + " extracted properly.", "TheQueue",
-          JobBuilder.extract(props2, JobConfPropertyNames.QUEUE_NAMES
-          .getCandidates(), null));
-      
-      // This config property is not interested for props2. So props should not
-      // contain this.
-      assertNull("Uninterested " + oldOrNew + " config property for job name "
-          + " is extracted.",
-          JobBuilder.extract(props2, JobConfPropertyNames.JOB_NAMES
+          JobBuilder.extract(props, JobConfPropertyNames.JOB_NAMES
           .getCandidates(), null));
 
-      validateChildJavaOpts(newConfig, props1);
-      validateChildJavaOpts(newConfig, props2);
+      validateChildJavaOpts(newConfig, props);
 
     } finally {
       inputConfStream.close();
@@ -575,6 +528,64 @@ public class TestRumenJobTraces {
           .getCandidates(), null));
     }
   }
+  
+    /**
+     * Test if the {@link JobConfigurationParser} can correctly extract out 
+     * key-value pairs from the job configuration.
+     */
+    @Test
+    public void testJobConfigurationParsing() throws Exception {
+      final FileSystem lfs = FileSystem.getLocal(new Configuration());
+  
+      final Path rootTempDir =
+          new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(
+              lfs.getUri(), lfs.getWorkingDirectory());
+  
+      final Path tempDir = new Path(rootTempDir, "TestJobConfigurationParser");
+      lfs.delete(tempDir, true);
+  
+      // Add some configuration parameters to the conf
+      JobConf jConf = new JobConf(false);
+      String key = "test.data";
+      String value = "hello world";
+      jConf.set(key, value);
+      
+      // create the job conf file
+      Path jobConfPath = new Path(tempDir.toString(), "job.xml");
+      lfs.delete(jobConfPath, false);
+      DataOutputStream jobConfStream = lfs.create(jobConfPath);
+      jConf.writeXml(jobConfStream);
+      jobConfStream.close();
+      
+      // now read the job conf file using the job configuration parser
+      Properties properties = 
+        JobConfigurationParser.parse(lfs.open(jobConfPath));
+      
+      // check if the required parameter is loaded
+      assertEquals("Total number of extracted properties (" + properties.size() 
+                   + ") doesn't match the expected size of 1 ["
+                   + "JobConfigurationParser]",
+                   1, properties.size());
+      // check if the key is present in the extracted configuration
+      assertTrue("Key " + key + " is missing in the configuration extracted "
+                 + "[JobConfigurationParser]",
+                 properties.keySet().contains(key));
+      // check if the desired property has the correct value
+      assertEquals("JobConfigurationParser couldn't recover the parameters"
+                   + " correctly",
+                  value, properties.get(key));
+      
+      // Test ZombieJob
+      LoggedJob job = new LoggedJob();
+      job.setJobProperties(properties);
+      
+      ZombieJob zjob = new ZombieJob(job, null);
+      Configuration zconf = zjob.getJobConf();
+      // check if the required parameter is loaded
+      assertEquals("ZombieJob couldn't recover the parameters correctly", 
+                   value, zconf.get(key));
+    }
+
 
   @Test
   public void testTopologyBuilder() throws Exception {

Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
(original)
+++ hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
Thu Apr 28 05:43:20 2011
@@ -3,6 +3,9 @@
   "user" : "hadoopqa",
   "jobName" : null,
   "jobID" : "job_200904211745_0002",
+   "jobProperties" : {
+    "mapred.child.java.opts" : "-server -Xmx640m -Djava.net.preferIPv4Stack=true"
+  },
   "mapTasks" : [ {
     "startTime" : 1240336753705,
     "attempts" : [ {

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
Thu Apr 28 05:43:20 2011
@@ -1594,6 +1594,8 @@ public class HadoopLogsAnalyzer extends 
       jobBeingTraced.setJobMapMB(jobconf.jobMapMB);
       jobBeingTraced.setJobReduceMB(jobconf.jobReduceMB);
 
+      jobBeingTraced.setJobProperties(jobconf.properties);
+      
       jobconf = null;
 
       finalizeJob();

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java Thu Apr
28 05:43:20 2011
@@ -96,6 +96,8 @@ public class JobBuilder {
   private static final Pattern heapPattern =
       Pattern.compile("-Xmx([0-9]+[kKmMgGtT])");
 
+  private Properties jobConfigurationParameters = null;
+
   public JobBuilder(String jobID) {
     this.jobID = jobID;
   }
@@ -228,6 +230,7 @@ public class JobBuilder {
           "JobBuilder.process(Properties conf) called after LoggedJob built");
     }
 
+    //TODO remove this once the deprecate APIs in LoggedJob are removed
     result.setQueue(extract(conf, JobConfPropertyNames.QUEUE_NAMES
         .getCandidates(), "default"));
     result.setJobName(extract(conf, JobConfPropertyNames.JOB_NAMES
@@ -239,6 +242,8 @@ public class JobBuilder {
         JobConfPropertyNames.MAP_JAVA_OPTS_S.getCandidates()));
     maybeSetJobReduceMB(extractMegabytes(conf,
         JobConfPropertyNames.REDUCE_JAVA_OPTS_S.getCandidates()));
+        
+    this.jobConfigurationParameters = conf;
   }
 
   /**
@@ -248,9 +253,12 @@ public class JobBuilder {
    * @return Parsed {@link LoggedJob} object.
    */
   public LoggedJob build() {
-    // The main job here is to build CDFs
+    // The main job here is to build CDFs and manage the conf
     finalized = true;
 
+    // set the conf
+    result.setJobProperties(jobConfigurationParameters);
+    
     // initialize all the per-job statistics gathering places
     Histogram[] successfulMapAttemptTimes =
         new Histogram[ParsedHost.numberOfDistances() + 1];

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
Thu Apr 28 05:43:20 2011
@@ -17,13 +17,9 @@
  */
 package org.apache.hadoop.tools.rumen;
 
-import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Properties;
-import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -38,22 +34,11 @@ import org.xml.sax.SAXException;
 
 /**
  * {@link JobConfigurationParser} parses the job configuration xml file, and
- * extracts various framework specific properties. It parses the file using a
+ * extracts configuration properties. It parses the file using a
  * stream-parser and thus is more memory efficient. [This optimization may be
  * postponed for a future release]
  */
 public class JobConfigurationParser {
-  final private Set<String> interested;
-
-  /**
-   * Constructor
-   * 
-   * @param interested
-   *          properties we should extract from the job configuration xml.
-   */
-  public JobConfigurationParser(List<String> interested) {
-    this.interested = new HashSet<String>(interested);
-  }
 
   /**
    * Parse the job configuration file (as an input stream) and return a
@@ -66,7 +51,7 @@ public class JobConfigurationParser {
    *         configuration xml.
    * @throws IOException
    */
-  Properties parse(InputStream input) throws IOException {
+  static Properties parse(InputStream input) throws IOException {
     Properties result = new Properties();
 
     try {
@@ -117,7 +102,7 @@ public class JobConfigurationParser {
           }
         }
 
-        if (interested.contains(attr) && value != null) {
+        if (attr != null && value != null) {
           result.put(attr, value);
         }
       }

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java Thu Apr
28 05:43:20 2011
@@ -22,6 +22,8 @@ package org.apache.hadoop.tools.rumen;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -92,6 +94,8 @@ public class LoggedJob implements DeepCo
   double[] mapperTriesToSucceed;
   double failedMapperFraction; // !!!!!
 
+  private Properties jobProperties = new Properties();
+  
   LoggedJob() {
 
   }
@@ -102,6 +106,20 @@ public class LoggedJob implements DeepCo
     setJobID(jobID);
   }
 
+  /**
+   * Set the configuration properties of the job.
+   */
+  void setJobProperties(Properties conf) {
+    this.jobProperties = conf;
+  }
+  
+  /**
+   * Get the configuration properties of the job.
+   */
+  public Properties getJobProperties() {
+    return jobProperties;
+  }
+  
   void adjustTimes(long adjustment) {
     submitTime += adjustment;
     launchTime += adjustment;
@@ -537,6 +555,35 @@ public class LoggedJob implements DeepCo
     }
   }
 
+  private void compareJobProperties(Properties prop1, Properties prop2,
+                                    TreePath loc, String eltname) 
+  throws DeepInequalityException {
+    if (prop1 == null && prop2 == null) {
+      return;
+    }
+
+    if (prop1 == null || prop2 == null) {
+      throw new DeepInequalityException(eltname + " miscompared [null]", 
+                                        new TreePath(loc, eltname));
+    }
+
+    if (prop1.size() != prop2.size()) {
+      throw new DeepInequalityException(eltname + " miscompared [size]", 
+                                        new TreePath(loc, eltname));
+    }
+    
+    for (Map.Entry<Object, Object> entry : prop1.entrySet()) {
+      Object v1 = entry.getValue();
+      Object v2 = prop2.get(entry.getKey());
+      if (v1 == null || v2 == null || !v1.equals(v2)) {
+        throw new DeepInequalityException(
+          eltname + " miscompared for value of key : " 
+            + entry.getKey().toString(), 
+          new TreePath(loc, eltname));
+      }
+    }
+  }
+  
   public void deepCompare(DeepCompare comparand, TreePath loc)
       throws DeepInequalityException {
     if (!(comparand instanceof LoggedJob)) {
@@ -600,5 +647,9 @@ public class LoggedJob implements DeepCo
     compare1(clusterReduceMB, other.clusterReduceMB, loc, "clusterReduceMB");
     compare1(jobMapMB, other.jobMapMB, loc, "jobMapMB");
     compare1(jobReduceMB, other.jobReduceMB, loc, "jobReduceMB");
+
+    // compare the job configuration parameters
+    compareJobProperties(jobProperties, other.getJobProperties(), loc, 
+                         "JobProperties");
   }
 }

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java Thu
Apr 28 05:43:20 2011
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.tools.rumen;
 
+import java.util.Properties;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 
@@ -57,6 +58,8 @@ class ParsedConfigFile {
   final String jobID;
 
   final boolean valid;
+  
+  final Properties properties = new Properties();
 
   private int maybeGetIntValue(String propName, String attr, String value,
       int oldValue) {
@@ -145,6 +148,8 @@ class ParsedConfigFile {
                 "true".equals(((Text) field.getFirstChild()).getData());
           }
         }
+        
+        properties.setProperty(attr, value);
 
         if ("mapred.child.java.opts".equals(attr) && value != null) {
           Matcher matcher = heapPattern.matcher(value);

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java Thu Apr
28 05:43:20 2011
@@ -50,7 +50,6 @@ public class TraceBuilder extends Config
   static final int RUN_METHOD_FAILED_EXIT_CODE = 3;
 
   TopologyBuilder topologyBuilder = new TopologyBuilder();
-  JobConfigurationParser jobConfParser;
   Outputter<LoggedJob> traceWriter;
   Outputter<LoggedNetworkTopology> topologyWriter;
 
@@ -191,25 +190,11 @@ public class TraceBuilder extends Config
     return jobId != null;
   }
 
-  private void addInterestedProperties(List<String> interestedProperties,
-      String[] names) {
-    for (String name : names) {
-      interestedProperties.add(name);
-    }
-  }
 
   @SuppressWarnings("unchecked")
   @Override
   public int run(String[] args) throws Exception {
     MyOptions options = new MyOptions(args, getConf());
-    List<String> interestedProperties = new ArrayList<String>();
-    {
-      for (JobConfPropertyNames candidateSet : JobConfPropertyNames.values()) {
-        addInterestedProperties(interestedProperties, candidateSet
-            .getCandidates());
-      }
-    }
-    jobConfParser = new JobConfigurationParser(interestedProperties);
     traceWriter = options.clazzTraceOutputter.newInstance();
     traceWriter.init(options.traceOutput, getConf());
     topologyWriter = new DefaultOutputter<LoggedNetworkTopology>();
@@ -254,7 +239,7 @@ public class TraceBuilder extends Config
               }
 
               if (isJobConfXml(filePair.first(), ris)) {
-                processJobConf(jobConfParser.parse(ris.rewind()), jobBuilder);
+            	processJobConf(JobConfigurationParser.parse(ris.rewind()), jobBuilder);
               } else {
                 parser = JobHistoryParserFactory.getParser(ris);
                 if (parser == null) {

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java?rev=1097325&r1=1097324&r2=1097325&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java Thu Apr
28 05:43:20 2011
@@ -120,8 +120,20 @@ public class ZombieJob implements JobSto
   @Override
   public synchronized JobConf getJobConf() {
     if (jobConf == null) {
-      // TODO : add more to jobConf ?
       jobConf = new JobConf();
+      
+      // Add parameters from the configuration in the job trace
+      //
+      // The reason why the job configuration parameters, as seen in the jobconf
+      // file, are added first because the specialized values obtained from 
+      // Rumen should override the job conf values.
+      //
+      for (Map.Entry<Object, Object> entry : job.getJobProperties().entrySet()) {
+        jobConf.set(entry.getKey().toString(), entry.getValue().toString());
+      }
+      
+      //TODO Eliminate parameters that are already copied from the job's 
+      // configuration file.
       jobConf.setJobName(getName());
       jobConf.setUser(getUser());
       jobConf.setNumMapTasks(getNumberMaps());



Mime
View raw message