hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079216 - in /hadoop/mapreduce/branches/yahoo-merge/src: test/mapred/org/apache/hadoop/tools/rumen/ test/tools/data/rumen/small-trace-test/ tools/org/apache/hadoop/tools/rumen/
Date Tue, 08 Mar 2011 05:57:02 GMT
Author: omalley
Date: Tue Mar  8 05:57:01 2011
New Revision: 1079216

URL: http://svn.apache.org/viewvc?rev=1079216&view=rev
Log:
commit 5f1b20b2190246305898dd9fd1af260583f818af
Author: Amar Ramesh Kamat <amarrk@yahoo-inc.com>
Date:   Mon Dec 13 19:08:21 2010 +0530

     Add job configuration parameters and their values to the trace generated by Rumen.
    
    +++ b/YAHOO-CHANGES.txt
    +   [MR-2153]: Add job configuration parameters and their values from
    +  the job configuration files to the trace generated by Rumen. Patch available
    +  at  (amarrk)
    +

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
    hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz
    hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
    hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
    hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
Tue Mar  8 05:57:01 2011
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.tools.rumen;
 
+import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -453,41 +454,13 @@ public class TestRumenJobTraces {
 
     // Validate parser with old mapred config properties from
     // sample-conf-file.xml
-    String[] oldProps1 = { "mapred.job.queue.name", "mapred.job.name",
-        "mapred.child.java.opts" };
+    validateJobConfParser("sample-conf.file.xml", false);
 
-    String[] oldProps2 = { "mapred.job.queue.name", "mapred.child.java.opts" };
-
-    validateJobConfParser(oldProps1, oldProps2, "sample-conf.file.xml", false);
-
-    // Validate parser with new mapreduce config properties from
-    // sample-conf-file.new.xml
-    String[] newProps1 = { MRJobConfig.QUEUE_NAME, MRJobConfig.JOB_NAME,
-        MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
-
-    String[] newProps2 = { MRJobConfig.QUEUE_NAME,
-        MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.REDUCE_JAVA_OPTS };
-
-    validateJobConfParser(newProps1, newProps2,
-        "sample-conf.file.new.xml", true);
+    validateJobConfParser("sample-conf.file.new.xml", true);
   }
 
-  private void validateJobConfParser(String[] list1, String[] list2,
-      String confFile, boolean newConfig)
+  private void validateJobConfParser(String confFile, boolean newConfig)
       throws Exception {
-    List<String> interested1 = new ArrayList<String>();
-    for (String interested : list1) {
-      interested1.add(interested);
-    }
-
-    List<String> interested2 = new ArrayList<String>();
-    for (String interested : list2) {
-      interested2.add(interested);
-    }
-
-    JobConfigurationParser jcp1 = new JobConfigurationParser(interested1);
-    JobConfigurationParser jcp2 = new JobConfigurationParser(interested2);
-
     final Configuration conf = new Configuration();
     final FileSystem lfs = FileSystem.getLocal(conf);
 
@@ -504,43 +477,20 @@ public class TestRumenJobTraces {
         new PossiblyDecompressedInputStream(inputPath, conf);
 
     try {
-      Properties props1 = jcp1.parse(inputConfStream);
+      Properties props = JobConfigurationParser.parse(inputConfStream);
       inputConfStream.close();
 
-      inputConfStream = new PossiblyDecompressedInputStream(inputPath, conf);
-      Properties props2 = jcp2.parse(inputConfStream);
-
-      assertEquals("testJobConfigurationParser: wrong number of properties",
-          list1.length, props1.size());
-      assertEquals("testJobConfigurationParser: wrong number of properties",
-          list2.length, props2.size());
-
-      // Make sure that parser puts the interested properties into props1 and
-      // props2 as defined by list1 and list2.
       String oldOrNew = newConfig ? "New" : "Old";
       assertEquals(oldOrNew + " config property for job queue name is not "
           + " extracted properly.", "TheQueue",
-          JobBuilder.extract(props1, JobConfPropertyNames.QUEUE_NAMES
+          JobBuilder.extract(props, JobConfPropertyNames.QUEUE_NAMES
           .getCandidates(), null));
       assertEquals(oldOrNew + " config property for job name is not "
           + " extracted properly.", "MyMRJob",
-          JobBuilder.extract(props1, JobConfPropertyNames.JOB_NAMES
+          JobBuilder.extract(props, JobConfPropertyNames.JOB_NAMES
           .getCandidates(), null));
 
-      assertEquals(oldOrNew + " config property for job queue name is not "
-          + " extracted properly.", "TheQueue",
-          JobBuilder.extract(props2, JobConfPropertyNames.QUEUE_NAMES
-          .getCandidates(), null));
-      
-      // This config property is not interested for props2. So props should not
-      // contain this.
-      assertNull("Uninterested " + oldOrNew + " config property for job name "
-          + " is extracted.",
-          JobBuilder.extract(props2, JobConfPropertyNames.JOB_NAMES
-          .getCandidates(), null));
-
-      validateChildJavaOpts(newConfig, props1);
-      validateChildJavaOpts(newConfig, props2);
+      validateChildJavaOpts(newConfig, props);
 
     } finally {
       inputConfStream.close();
@@ -586,6 +536,63 @@ public class TestRumenJobTraces {
     }
   }
 
+  /**
+   * Test if the {@link JobConfigurationParser} can correctly extract out 
+   * key-value pairs from the job configuration.
+   */
+  @Test
+  public void testJobConfigurationParsing() throws Exception {
+    final FileSystem lfs = FileSystem.getLocal(new Configuration());
+
+    final Path rootTempDir =
+        new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(
+            lfs.getUri(), lfs.getWorkingDirectory());
+
+    final Path tempDir = new Path(rootTempDir, "TestJobConfigurationParser");
+    lfs.delete(tempDir, true);
+
+    // Add some configuration parameters to the conf
+    JobConf jConf = new JobConf(false);
+    String key = "test.data";
+    String value = "hello world";
+    jConf.set(key, value);
+    
+    // create the job conf file
+    Path jobConfPath = new Path(tempDir.toString(), "job.xml");
+    lfs.delete(jobConfPath, false);
+    DataOutputStream jobConfStream = lfs.create(jobConfPath);
+    jConf.writeXml(jobConfStream);
+    jobConfStream.close();
+    
+    // now read the job conf file using the job configuration parser
+    Properties properties = 
+      JobConfigurationParser.parse(lfs.open(jobConfPath));
+    
+    // check if the required parameter is loaded
+    assertEquals("Total number of extracted properties (" + properties.size() 
+                 + ") doesn't match the expected size of 1 ["
+                 + "JobConfigurationParser]",
+                 1, properties.size());
+    // check if the key is present in the extracted configuration
+    assertTrue("Key " + key + " is missing in the configuration extracted "
+               + "[JobConfigurationParser]",
+               properties.keySet().contains(key));
+    // check if the desired property has the correct value
+    assertEquals("JobConfigurationParser couldn't recover the parameters"
+                 + " correctly",
+                 value, properties.get(key));
+    
+    // Test ZombieJob
+    LoggedJob job = new LoggedJob();
+    job.setJobProperties(properties);
+    
+    ZombieJob zjob = new ZombieJob(job, null);
+    Configuration zconf = zjob.getJobConf();
+    // check if the required parameter is loaded
+    assertEquals("ZombieJob couldn't recover the parameters correctly", 
+                 value, zconf.get(key));
+  }
+
   @Test
   public void testTopologyBuilder() throws Exception {
     final TopologyBuilder subject = new TopologyBuilder();

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/counters-test-trace.json.gz?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/dispatch-trace-output.json.gz?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/truncated-trace-output?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/tools/data/rumen/small-trace-test/truncated-trace-output
Tue Mar  8 05:57:01 2011
@@ -3,6 +3,9 @@
   "user" : "hadoopqa",
   "jobName" : null,
   "jobID" : "job_200904211745_0002",
+   "jobProperties" : {
+    "mapred.child.java.opts" : "-server -Xmx640m -Djava.net.preferIPv4Stack=true"
+  },
   "mapTasks" : [ {
     "startTime" : 1240336753705,
     "attempts" : [ {

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/HadoopLogsAnalyzer.java
Tue Mar  8 05:57:01 2011
@@ -1594,6 +1594,8 @@ public class HadoopLogsAnalyzer extends 
       jobBeingTraced.setJobMapMB(jobconf.jobMapMB);
       jobBeingTraced.setJobReduceMB(jobconf.jobReduceMB);
 
+      jobBeingTraced.setJobProperties(jobconf.properties);
+      
       jobconf = null;
 
       finalizeJob();

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobBuilder.java
Tue Mar  8 05:57:01 2011
@@ -96,6 +96,8 @@ public class JobBuilder {
   private static final Pattern heapPattern =
       Pattern.compile("-Xmx([0-9]+[kKmMgGtT])");
 
+  private Properties jobConfigurationParameters = null;
+
   public JobBuilder(String jobID) {
     this.jobID = jobID;
   }
@@ -228,6 +230,7 @@ public class JobBuilder {
           "JobBuilder.process(Properties conf) called after LoggedJob built");
     }
 
+    //TODO remove this once the deprecate APIs in LoggedJob are removed
     result.setQueue(extract(conf, JobConfPropertyNames.QUEUE_NAMES
         .getCandidates(), "default"));
     result.setJobName(extract(conf, JobConfPropertyNames.JOB_NAMES
@@ -239,6 +242,8 @@ public class JobBuilder {
         JobConfPropertyNames.MAP_JAVA_OPTS_S.getCandidates()));
     maybeSetJobReduceMB(extractMegabytes(conf,
         JobConfPropertyNames.REDUCE_JAVA_OPTS_S.getCandidates()));
+        
+    this.jobConfigurationParameters = conf;
   }
 
   /**
@@ -248,9 +253,12 @@ public class JobBuilder {
    * @return Parsed {@link LoggedJob} object.
    */
   public LoggedJob build() {
-    // The main job here is to build CDFs
+    // The main job here is to build CDFs and manage the conf
     finalized = true;
 
+    // set the conf
+    result.setJobProperties(jobConfigurationParameters);
+    
     // initialize all the per-job statistics gathering places
     Histogram[] successfulMapAttemptTimes =
         new Histogram[ParsedHost.numberOfDistances() + 1];

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/JobConfigurationParser.java
Tue Mar  8 05:57:01 2011
@@ -17,13 +17,9 @@
  */
 package org.apache.hadoop.tools.rumen;
 
-import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Properties;
-import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -38,22 +34,11 @@ import org.xml.sax.SAXException;
 
 /**
  * {@link JobConfigurationParser} parses the job configuration xml file, and
- * extracts various framework specific properties. It parses the file using a
+ * extracts configuration properties. It parses the file using a
  * stream-parser and thus is more memory efficient. [This optimization may be
  * postponed for a future release]
  */
 public class JobConfigurationParser {
-  final private Set<String> interested;
-
-  /**
-   * Constructor
-   * 
-   * @param interested
-   *          properties we should extract from the job configuration xml.
-   */
-  public JobConfigurationParser(List<String> interested) {
-    this.interested = new HashSet<String>(interested);
-  }
 
   /**
    * Parse the job configuration file (as an input stream) and return a
@@ -66,7 +51,7 @@ public class JobConfigurationParser {
    *         configuration xml.
    * @throws IOException
    */
-  Properties parse(InputStream input) throws IOException {
+  static Properties parse(InputStream input) throws IOException {
     Properties result = new Properties();
 
     try {
@@ -117,7 +102,7 @@ public class JobConfigurationParser {
           }
         }
 
-        if (interested.contains(attr) && value != null) {
+        if (attr != null && value != null) {
           result.put(attr, value);
         }
       }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
Tue Mar  8 05:57:01 2011
@@ -22,6 +22,8 @@ package org.apache.hadoop.tools.rumen;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 import java.util.TreeSet;
 
@@ -92,6 +94,8 @@ public class LoggedJob implements DeepCo
   double[] mapperTriesToSucceed;
   double failedMapperFraction; // !!!!!
 
+  private Properties jobProperties = new Properties();
+  
   LoggedJob() {
 
   }
@@ -102,6 +106,20 @@ public class LoggedJob implements DeepCo
     setJobID(jobID);
   }
 
+  /**
+   * Set the configuration properties of the job.
+   */
+  void setJobProperties(Properties conf) {
+    this.jobProperties = conf;
+  }
+  
+  /**
+   * Get the configuration properties of the job.
+   */
+  public Properties getJobProperties() {
+    return jobProperties;
+  }
+  
   void adjustTimes(long adjustment) {
     submitTime += adjustment;
     launchTime += adjustment;
@@ -211,10 +229,12 @@ public class LoggedJob implements DeepCo
     this.finishTime = finishTime;
   }
 
+  @Deprecated
   public int getHeapMegabytes() {
     return heapMegabytes;
   }
 
+  @Deprecated
   void setHeapMegabytes(int heapMegabytes) {
     this.heapMegabytes = heapMegabytes;
   }
@@ -341,6 +361,7 @@ public class LoggedJob implements DeepCo
     this.relativeTime = relativeTime;
   }
 
+  @Deprecated
   public String getQueue() {
     return queue;
   }
@@ -357,34 +378,42 @@ public class LoggedJob implements DeepCo
     this.jobName = jobName;
   }
 
+  @Deprecated
   public int getClusterMapMB() {
     return clusterMapMB;
   }
 
+  @Deprecated
   void setClusterMapMB(int clusterMapMB) {
     this.clusterMapMB = clusterMapMB;
   }
 
+  @Deprecated
   public int getClusterReduceMB() {
     return clusterReduceMB;
   }
 
+  @Deprecated
   void setClusterReduceMB(int clusterReduceMB) {
     this.clusterReduceMB = clusterReduceMB;
   }
 
+  @Deprecated
   public int getJobMapMB() {
     return jobMapMB;
   }
 
+  @Deprecated
   void setJobMapMB(int jobMapMB) {
     this.jobMapMB = jobMapMB;
   }
 
+  @Deprecated
   public int getJobReduceMB() {
     return jobReduceMB;
   }
 
+  @Deprecated
   void setJobReduceMB(int jobReduceMB) {
     this.jobReduceMB = jobReduceMB;
   }
@@ -537,6 +566,35 @@ public class LoggedJob implements DeepCo
     }
   }
 
+  private void compareJobProperties(Properties prop1, Properties prop2,
+                                    TreePath loc, String eltname) 
+  throws DeepInequalityException {
+    if (prop1 == null && prop2 == null) {
+      return;
+    }
+
+    if (prop1 == null || prop2 == null) {
+      throw new DeepInequalityException(eltname + " miscompared [null]", 
+                                        new TreePath(loc, eltname));
+    }
+
+    if (prop1.size() != prop2.size()) {
+      throw new DeepInequalityException(eltname + " miscompared [size]", 
+                                        new TreePath(loc, eltname));
+    }
+    
+    for (Map.Entry<Object, Object> entry : prop1.entrySet()) {
+      Object v1 = entry.getValue();
+      Object v2 = prop2.get(entry.getKey());
+      if (v1 == null || v2 == null || !v1.equals(v2)) {
+        throw new DeepInequalityException(
+          eltname + " miscompared for value of key : " 
+            + entry.getKey().toString(), 
+          new TreePath(loc, eltname));
+      }
+    }
+  }
+  
   public void deepCompare(DeepCompare comparand, TreePath loc)
       throws DeepInequalityException {
     if (!(comparand instanceof LoggedJob)) {
@@ -600,5 +658,9 @@ public class LoggedJob implements DeepCo
     compare1(clusterReduceMB, other.clusterReduceMB, loc, "clusterReduceMB");
     compare1(jobMapMB, other.jobMapMB, loc, "jobMapMB");
     compare1(jobReduceMB, other.jobReduceMB, loc, "jobReduceMB");
+
+    // compare the job configuration parameters
+    compareJobProperties(jobProperties, other.getJobProperties(), loc, 
+                         "JobProperties");
   }
 }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ParsedConfigFile.java
Tue Mar  8 05:57:01 2011
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.tools.rumen;
 
+import java.util.Properties;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 
@@ -57,6 +58,8 @@ class ParsedConfigFile {
   final String jobID;
 
   final boolean valid;
+  
+  final Properties properties = new Properties();
 
   private int maybeGetIntValue(String propName, String attr, String value,
       int oldValue) {
@@ -145,6 +148,8 @@ class ParsedConfigFile {
                 "true".equals(((Text) field.getFirstChild()).getData());
           }
         }
+        
+        properties.setProperty(attr, value);
 
         if ("mapred.child.java.opts".equals(attr) && value != null) {
           Matcher matcher = heapPattern.matcher(value);

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/TraceBuilder.java
Tue Mar  8 05:57:01 2011
@@ -50,7 +50,6 @@ public class TraceBuilder extends Config
   static final int RUN_METHOD_FAILED_EXIT_CODE = 3;
 
   TopologyBuilder topologyBuilder = new TopologyBuilder();
-  JobConfigurationParser jobConfParser;
   Outputter<LoggedJob> traceWriter;
   Outputter<LoggedNetworkTopology> topologyWriter;
 
@@ -189,25 +188,10 @@ public class TraceBuilder extends Config
     return jobId != null;
   }
 
-  private void addInterestedProperties(List<String> interestedProperties,
-      String[] names) {
-    for (String name : names) {
-      interestedProperties.add(name);
-    }
-  }
-
   @SuppressWarnings("unchecked")
   @Override
   public int run(String[] args) throws Exception {
     MyOptions options = new MyOptions(args, getConf());
-    List<String> interestedProperties = new ArrayList<String>();
-    {
-      for (JobConfPropertyNames candidateSet : JobConfPropertyNames.values()) {
-        addInterestedProperties(interestedProperties, candidateSet
-            .getCandidates());
-      }
-    }
-    jobConfParser = new JobConfigurationParser(interestedProperties);
     traceWriter = options.clazzTraceOutputter.newInstance();
     traceWriter.init(options.traceOutput, getConf());
     topologyWriter = new DefaultOutputter<LoggedNetworkTopology>();
@@ -252,7 +236,7 @@ public class TraceBuilder extends Config
               }
 
               if (isJobConfXml(filePair.first(), ris)) {
-                processJobConf(jobConfParser.parse(ris.rewind()), jobBuilder);
+                processJobConf(JobConfigurationParser.parse(ris.rewind()), jobBuilder);
               } else {
                 parser = JobHistoryParserFactory.getParser(ris);
                 if (parser == null) {

Modified: hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java?rev=1079216&r1=1079215&r2=1079216&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/tools/org/apache/hadoop/tools/rumen/ZombieJob.java
Tue Mar  8 05:57:01 2011
@@ -120,8 +120,20 @@ public class ZombieJob implements JobSto
   @Override
   public synchronized JobConf getJobConf() {
     if (jobConf == null) {
-      // TODO : add more to jobConf ?
       jobConf = new JobConf();
+      
+      // Add parameters from the configuration in the job trace
+      //
+      // The reason why the job configuration parameters, as seen in the jobconf
+      // file, are added first because the specialized values obtained from 
+      // Rumen should override the job conf values.
+      //
+      for (Map.Entry<Object, Object> entry : job.getJobProperties().entrySet()) {
+        jobConf.set(entry.getKey().toString(), entry.getValue().toString());
+      }
+      
+      //TODO Eliminate parameters that are already copied from the job's 
+      // configuration file.
       jobConf.setJobName(getName());
       jobConf.setUser(getUser());
       jobConf.setNumMapTasks(getNumberMaps());



Mime
View raw message