hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r826508 - in /hadoop/mapreduce/trunk: ./ src/test/mapred/org/apache/hadoop/tools/rumen/ src/test/tools/data/rumen/small-trace-test/ src/tools/org/apache/hadoop/tools/rumen/
Date Sun, 18 Oct 2009 19:38:39 GMT
Author: cdouglas
Date: Sun Oct 18 19:38:37 2009
New Revision: 826508

URL: http://svn.apache.org/viewvc?rev=826508&view=rev
Log:
MAPREDUCE-1077. Fix Rumen so that truncated tasks do not mark the job as
successful. Contributed by Dick King

Added:
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
  (with props)
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz
  (with props)
Removed:
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output
    hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java
    hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Sun Oct 18 19:38:37 2009
@@ -782,3 +782,6 @@
 
     MAPREDUCE-1061. Add unit test validating byte specifications for gridmix
     jobs. (cdouglas)
+
+    MAPREDUCE-1077. Fix Rumen so that truncated tasks do not mark the job as
+    successful. (Dick King via cdouglas)

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
(original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
Sun Oct 18 19:38:37 2009
@@ -33,8 +33,8 @@
 public class TestRumenJobTraces {
   @Test
   public void testSmallTrace() throws Exception {
-    performSingleTest("sample-job-tracker-logs",
-        "job-tracker-logs-topology-output", "job-tracker-logs-trace-output");
+    performSingleTest("sample-job-tracker-logs.gz",
+        "job-tracker-logs-topology-output", "job-tracker-logs-trace-output.gz");
   }
 
   @Test
@@ -96,9 +96,8 @@
   static private <T extends DeepCompare> void jsonFileMatchesGold(
       FileSystem lfs, Path result, Path gold, Class<? extends T> clazz,
       String fileDescription) throws IOException {
-    InputStream goldStream = lfs.open(gold);
     JsonObjectMapperParser<T> goldParser =
-        new JsonObjectMapperParser<T>(goldStream, clazz);
+        new JsonObjectMapperParser<T>(gold, clazz, new Configuration());
     InputStream resultStream = lfs.open(result);
     JsonObjectMapperParser<T> resultParser =
         new JsonObjectMapperParser<T>(resultStream, clazz);

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz?rev=826508&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz?rev=826508&view=auto
==============================================================================
Binary file - no diff available.

Propchange: hadoop/mapreduce/trunk/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java Sun Oct
18 19:38:37 2009
@@ -22,6 +22,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.TreeSet;
 
 import org.codehaus.jackson.annotate.JsonAnySetter;
@@ -44,7 +45,8 @@
     VERY_LOW, LOW, NORMAL, HIGH, VERY_HIGH
   };
 
-  static private TreeSet<String> alreadySeenAnySetterAttributes = new TreeSet<String>();
+  static private Set<String> alreadySeenAnySetterAttributes =
+      new TreeSet<String>();
 
   String jobID;
   String user;
@@ -59,7 +61,7 @@
   int heapMegabytes = -1;
   int totalMaps = -1;
   int totalReduces = -1;
-  Pre21JobHistoryConstants.Values outcome = Pre21JobHistoryConstants.Values.SUCCESS;
+  Pre21JobHistoryConstants.Values outcome = null;
   JobType jobtype = JobType.JAVA;
   JobPriority priority = JobPriority.NORMAL;
 
@@ -100,7 +102,8 @@
     setJobID(jobID);
   }
 
-  @SuppressWarnings("unused") // for input parameter ignored.
+  @SuppressWarnings("unused")
+  // for input parameter ignored.
   @JsonAnySetter
   public void setUnknownAttribute(String attributeName, Object ignored) {
     if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
@@ -388,8 +391,9 @@
     }
   }
 
-  private void compare1(Pre21JobHistoryConstants.Values c1, Pre21JobHistoryConstants.Values
c2,
-      TreePath loc, String eltname) throws DeepInequalityException {
+  private void compare1(Pre21JobHistoryConstants.Values c1,
+      Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+      throws DeepInequalityException {
     if (c1 != c2) {
       throw new DeepInequalityException(eltname + " miscompared", new TreePath(
           loc, eltname));

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java Sun
Oct 18 19:38:37 2009
@@ -19,6 +19,10 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
 
 /**
  * A {@link LoggedLocation} is a representation of a point in an hierarchical
@@ -41,9 +45,12 @@
    * The full path from the root of the network to the host.
    * 
    * NOTE that this assumes that the network topology is a tree.
-   */ 
+   */
   List<String> layers = new ArrayList<String>();
 
+  static private Set<String> alreadySeenAnySetterAttributes =
+      new TreeSet<String>();
+
   public List<String> getLayers() {
     return layers;
   }
@@ -52,6 +59,17 @@
     this.layers = layers;
   }
 
+  @SuppressWarnings("unused")
+  // for input parameter ignored.
+  @JsonAnySetter
+  public void setUnknownAttribute(String attributeName, Object ignored) {
+    if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+      alreadySeenAnySetterAttributes.add(attributeName);
+      System.err.println("In LoggedJob, we saw the unknown attribute "
+          + attributeName + ".");
+    }
+  }
+
   // I'll treat this as an atomic object type
   private void compareStrings(List<String> c1, List<String> c2, TreePath loc,
       String eltname) throws DeepInequalityException {

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
Sun Oct 18 19:38:37 2009
@@ -23,9 +23,13 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.ArrayList;
 import java.util.Comparator;
 
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
 /**
  * A {@link LoggedNetworkTopology} represents a tree that in turn represents a
  * hierarchy of hosts. The current version requires the tree to have all leaves
@@ -39,10 +43,24 @@
   String name;
   List<LoggedNetworkTopology> children = new ArrayList<LoggedNetworkTopology>();
 
+  static private Set<String> alreadySeenAnySetterAttributes =
+      new TreeSet<String>();
+
   public LoggedNetworkTopology() {
     super();
   }
 
+  @SuppressWarnings("unused")
+  // for input parameter ignored.
+  @JsonAnySetter
+  public void setUnknownAttribute(String attributeName, Object ignored) {
+    if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+      alreadySeenAnySetterAttributes.add(attributeName);
+      System.err.println("In LoggedJob, we saw the unknown attribute "
+          + attributeName + ".");
+    }
+  }
+
   /**
    * We need this because we have to sort the {@code children} field. That field
    * is set-valued, but if we sort these fields we ensure that comparisons won't
@@ -70,7 +88,8 @@
     this.children = null;
 
     if (level < ParsedHost.numberOfDistances() - 1) {
-      HashMap<String, HashSet<ParsedHost>> topologies = new HashMap<String,
HashSet<ParsedHost>>();
+      HashMap<String, HashSet<ParsedHost>> topologies =
+          new HashMap<String, HashSet<ParsedHost>>();
 
       Iterator<ParsedHost> iter = hosts.iterator();
 

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
Sun Oct 18 19:38:37 2009
@@ -17,6 +17,11 @@
  */
 package org.apache.hadoop.tools.rumen;
 
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
 /**
  * A {@link LoggedSingleRelativeRanking} represents an X-Y coordinate of a
  * single point in a discrete CDF.
@@ -36,6 +41,20 @@
    */
   long datum = -1L;
 
+  static private Set<String> alreadySeenAnySetterAttributes =
+      new TreeSet<String>();
+
+  @SuppressWarnings("unused")
+  // for input parameter ignored.
+  @JsonAnySetter
+  public void setUnknownAttribute(String attributeName, Object ignored) {
+    if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+      alreadySeenAnySetterAttributes.add(attributeName);
+      System.err.println("In LoggedJob, we saw the unknown attribute "
+          + attributeName + ".");
+    }
+  }
+
   public double getRelativeRanking() {
     return relativeRanking;
   }

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java Sun Oct
18 19:38:37 2009
@@ -19,6 +19,10 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
 
 /**
  * A {@link LoggedTask} represents a [hadoop] task that is part of a hadoop job.
@@ -41,11 +45,26 @@
   Pre21JobHistoryConstants.Values taskStatus;
   List<LoggedTaskAttempt> attempts = new ArrayList<LoggedTaskAttempt>();
 
-  ArrayList<LoggedLocation> preferredLocations = new ArrayList<LoggedLocation>();
+  ArrayList<LoggedLocation> preferredLocations =
+      new ArrayList<LoggedLocation>();
 
   int numberMaps = -1;
   int numberReduces = -1;
 
+  static private Set<String> alreadySeenAnySetterAttributes =
+      new TreeSet<String>();
+
+  @SuppressWarnings("unused")
+  // for input parameter ignored.
+  @JsonAnySetter
+  public void setUnknownAttribute(String attributeName, Object ignored) {
+    if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+      alreadySeenAnySetterAttributes.add(attributeName);
+      System.err.println("In LoggedJob, we saw the unknown attribute "
+          + attributeName + ".");
+    }
+  }
+
   LoggedTask() {
     super();
   }
@@ -173,8 +192,9 @@
     }
   }
 
-  private void compare1(Pre21JobHistoryConstants.Values c1, Pre21JobHistoryConstants.Values
c2,
-      TreePath loc, String eltname) throws DeepInequalityException {
+  private void compare1(Pre21JobHistoryConstants.Values c1,
+      Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+      throws DeepInequalityException {
     if (c1 == null && c2 == null) {
       return;
     }

Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java?rev=826508&r1=826507&r2=826508&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
(original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
Sun Oct 18 19:38:37 2009
@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.tools.rumen;
 
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
 // HACK ALERT!!!  This "should" have have two subclasses, which might be called
 //                LoggedMapTaskAttempt and LoggedReduceTaskAttempt, but 
 //                the Jackson implementation of JSON doesn't handle a 
@@ -63,6 +68,20 @@
     super();
   }
 
+  static private Set<String> alreadySeenAnySetterAttributes =
+      new TreeSet<String>();
+
+  @SuppressWarnings("unused")
+  // for input parameter ignored.
+  @JsonAnySetter
+  public void setUnknownAttribute(String attributeName, Object ignored) {
+    if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+      alreadySeenAnySetterAttributes.add(attributeName);
+      System.err.println("In LoggedJob, we saw the unknown attribute "
+          + attributeName + ".");
+    }
+  }
+
   public long getShuffleFinished() {
     return shuffleFinished;
   }
@@ -259,8 +278,9 @@
     }
   }
 
-  private void compare1(Pre21JobHistoryConstants.Values c1, Pre21JobHistoryConstants.Values
c2,
-      TreePath loc, String eltname) throws DeepInequalityException {
+  private void compare1(Pre21JobHistoryConstants.Values c1,
+      Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+      throws DeepInequalityException {
     if (c1 != c2) {
       throw new DeepInequalityException(eltname + " miscompared", new TreePath(
           loc, eltname));



Mime
View raw message