hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077507 - in /hadoop/common/branches/branch-0.20-security-patches/src: core/org/apache/hadoop/util/ mapred/org/apache/hadoop/mapred/ test/org/apache/hadoop/util/
Date Fri, 04 Mar 2011 04:21:52 GMT
Author: omalley
Date: Fri Mar  4 04:21:52 2011
New Revision: 1077507

URL: http://svn.apache.org/viewvc?rev=1077507&view=rev
Log:
commit e03a692508fccd193bd6add616214be7558f6fe0
Author: Luke Lu <llu@yahoo-inc.com>
Date:   Tue Mar 23 10:13:46 2010 -0700

    MAPREDUCE-1545: from https://issues.apache.org/jira/secure/attachment/12447725/mr-1545-y20s-v4.patch

Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/StringUtils.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestStringUtils.java

Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/StringUtils.java?rev=1077507&r1=1077506&r2=1077507&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/StringUtils.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/util/StringUtils.java
Fri Mar  4 04:21:52 2011
@@ -709,4 +709,33 @@ public class StringUtils {
     }
     return sb.toString();
   }
+
+  /**
+   * Capitalize a word
+   *
+   * @param s the input string
+   * @return capitalized string
+   */
+  public static String capitalize(String s) {
+    int len = s.length();
+    if (len == 0) return s;
+    return new StringBuilder(len).append(Character.toTitleCase(s.charAt(0)))
+                                 .append(s.substring(1)).toString();
+  }
+
+  /**
+   * Convert SOME_STUFF to SomeStuff
+   *
+   * @param s input string
+   * @return camelized string
+   */
+  public static String camelize(String s) {
+    StringBuilder sb = new StringBuilder();
+    String[] words = split(s.toLowerCase(Locale.US), ESCAPE_CHAR, '_');
+
+    for (String word : words)
+      sb.append(capitalize(word));
+
+    return sb.toString();
+  }
 }

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1077507&r1=1077506&r2=1077507&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Fri Mar  4 04:21:52 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
@@ -41,7 +42,6 @@ import org.apache.hadoop.fs.LocalFileSys
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.CleanupQueue.PathDeletionContext;
-import org.apache.hadoop.mapred.AuditLogger;
 import org.apache.hadoop.mapred.JobHistory.Values;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobSubmissionFiles;
@@ -199,6 +199,10 @@ public class JobInProgress {
   long startTime;
   long launchTime;
   long finishTime;
+
+  // First *task launch time
+  final Map<TaskType, Long> firstTaskLaunchTimes =
+      new EnumMap<TaskType, Long>(TaskType.class);
   
   // Indicates how many times the job got restarted
   private final int restartCount;
@@ -718,6 +722,9 @@ public class JobInProgress {
   public synchronized long getLaunchTime() {
     return launchTime;
   }
+  Map<TaskType, Long> getFirstTaskLaunchTimes() {
+    return firstTaskLaunchTimes;
+  }
   public long getStartTime() {
     return startTime;
   }
@@ -1536,6 +1543,7 @@ public class JobInProgress {
     if (tip.isFirstAttempt(id)) {
       JobHistory.Task.logStarted(tip.getTIPId(), name,
                                  tip.getExecStartTime(), splits);
+      setFirstTaskLaunchTime(tip);
     }
     if (!tip.isJobSetupTask() && !tip.isJobCleanupTask()) {
       jobCounters.incrCounter(counter, 1);
@@ -1590,7 +1598,18 @@ public class JobInProgress {
       }
     }
   }
-    
+
+  void setFirstTaskLaunchTime(TaskInProgress tip) {
+    TaskType key = tip.getFirstTaskType();
+
+    synchronized(firstTaskLaunchTimes) {
+      // Could be optimized to do only one lookup with a little more code
+      if (!firstTaskLaunchTimes.containsKey(key)) {
+        firstTaskLaunchTimes.put(key, tip.getExecStartTime());
+      }
+    }
+  }
+
   static String convertTrackerNameToHostName(String trackerName) {
     // Ugly!
     // Convert the trackerName to it's host name
@@ -3133,6 +3152,49 @@ public class JobInProgress {
     static final char[] charsToEscape = 
       {StringUtils.COMMA, EQUALS, StringUtils.ESCAPE_CHAR};
     
+    static class SummaryBuilder {
+      final StringBuilder buffer = new StringBuilder();
+
+      // A little optimization for a very common case
+      SummaryBuilder add(String key, long value) {
+        return _add(key, Long.toString(value));
+      }
+
+      <T> SummaryBuilder add(String key, T value) {
+        return _add(key, StringUtils.escapeString(String.valueOf(value),
+                    StringUtils.ESCAPE_CHAR, charsToEscape));
+      }
+
+      SummaryBuilder add(SummaryBuilder summary) {
+        if (buffer.length() > 0) buffer.append(StringUtils.COMMA);
+        buffer.append(summary.buffer);
+        return this;
+      }
+
+      SummaryBuilder _add(String key, String value) {
+        if (buffer.length() > 0) buffer.append(StringUtils.COMMA);
+        buffer.append(key).append(EQUALS).append(value);
+        return this;
+      }
+
+      @Override public String toString() {
+        return buffer.toString();
+      }
+    }
+
+    static SummaryBuilder getTaskLaunchTimesSummary(JobInProgress job) {
+      SummaryBuilder summary = new SummaryBuilder();
+      Map<TaskType, Long> timeMap = job.getFirstTaskLaunchTimes();
+
+      synchronized(timeMap) {
+        for (Map.Entry<TaskType, Long> e : timeMap.entrySet()) {
+          summary.add("first"+ StringUtils.camelize(e.getKey().name()) +
+                      "TaskLaunchTime", e.getValue().longValue());
+        }
+      }
+      return summary;
+    }
+
     /**
      * Log a summary of the job's runtime.
      * 
@@ -3144,12 +3206,6 @@ public class JobInProgress {
     public static void logJobSummary(JobInProgress job, ClusterStatus cluster) {
       JobStatus status = job.getStatus();
       JobProfile profile = job.getProfile();
-      String user = StringUtils.escapeString(profile.getUser(), 
-                                             StringUtils.ESCAPE_CHAR, 
-                                             charsToEscape);
-      String queue = StringUtils.escapeString(profile.getQueueName(), 
-                                              StringUtils.ESCAPE_CHAR, 
-                                              charsToEscape);
       Counters jobCounters = job.getJobCounters();
       long mapSlotSeconds = 
         (jobCounters.getCounter(Counter.SLOTS_MILLIS_MAPS) +
@@ -3158,30 +3214,25 @@ public class JobInProgress {
         (jobCounters.getCounter(Counter.SLOTS_MILLIS_REDUCES) +
          jobCounters.getCounter(Counter.FALLOW_SLOTS_MILLIS_REDUCES)) / 1000;
 
-      LOG.info("jobId=" + job.getJobID() + StringUtils.COMMA +
-               "submitTime" + EQUALS + job.getStartTime() + StringUtils.COMMA +
-               "launchTime" + EQUALS + job.getLaunchTime() + StringUtils.COMMA +
-               "finishTime" + EQUALS + job.getFinishTime() + StringUtils.COMMA +
-               "numMaps" + EQUALS + job.getTasks(TaskType.MAP).length + 
-                           StringUtils.COMMA +
-               "numSlotsPerMap" + EQUALS + job.getNumSlotsPerMap() + 
-                                  StringUtils.COMMA +
-               "numReduces" + EQUALS + job.getTasks(TaskType.REDUCE).length + 
-                              StringUtils.COMMA +
-               "numSlotsPerReduce" + EQUALS + job.getNumSlotsPerReduce() + 
-                                     StringUtils.COMMA +
-               "user" + EQUALS + user + StringUtils.COMMA +
-               "queue" + EQUALS + queue + StringUtils.COMMA +
-               "status" + EQUALS + 
-                          JobStatus.getJobRunState(status.getRunState()) + 
-                          StringUtils.COMMA + 
-               "mapSlotSeconds" + EQUALS + mapSlotSeconds + StringUtils.COMMA +
-               "reduceSlotsSeconds" + EQUALS + reduceSlotSeconds  + 
-                                      StringUtils.COMMA +
-               "clusterMapCapacity" + EQUALS + cluster.getMaxMapTasks() + 
-                                      StringUtils.COMMA +
-               "clusterReduceCapacity" + EQUALS + cluster.getMaxReduceTasks()
-      );
+      SummaryBuilder summary = new SummaryBuilder()
+          .add("jobId", job.getJobID())
+          .add("submitTime", job.getStartTime())
+          .add("launchTime", job.getLaunchTime())
+          .add(getTaskLaunchTimesSummary(job))
+          .add("finishTime", job.getFinishTime())
+          .add("numMaps", job.getTasks(TaskType.MAP).length)
+          .add("numSlotsPerMap", job.getNumSlotsPerMap())
+          .add("numReduces", job.getTasks(TaskType.REDUCE).length)
+          .add("numSlotsPerReduce", job.getNumSlotsPerReduce())
+          .add("user", profile.getUser())
+          .add("queue", profile.getQueueName())
+          .add("status", JobStatus.getJobRunState(status.getRunState()))
+          .add("mapSlotSeconds", mapSlotSeconds)
+          .add("reduceSlotsSeconds", reduceSlotSeconds)
+          .add("clusterMapCapacity", cluster.getMaxMapTasks())
+          .add("clusterReduceCapacity", cluster.getMaxReduceTasks());
+
+      LOG.info(summary);
     }
   }
 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java?rev=1077507&r1=1077506&r2=1077507&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskInProgress.java
Fri Mar  4 04:21:52 2011
@@ -316,6 +316,11 @@ class TaskInProgress {
       return TaskType.REDUCE;
     }
   }
+
+  TaskType getFirstTaskType() {
+    assert firstTaskId != null : "got first task";
+    return getAttemptType(firstTaskId);
+  }
   
   /**
    * Is the Task associated with taskid is the first attempt of the tip? 

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestStringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestStringUtils.java?rev=1077507&r1=1077506&r2=1077507&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestStringUtils.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/util/TestStringUtils.java
Fri Mar  4 04:21:52 2011
@@ -132,4 +132,39 @@ public class TestStringUtils extends Tes
     assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
     assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
   }
+
+  public void testCamelize() {
+    // common use cases
+    assertEquals("Map", StringUtils.camelize("MAP"));
+    assertEquals("JobSetup", StringUtils.camelize("JOB_SETUP"));
+    assertEquals("SomeStuff", StringUtils.camelize("some_stuff"));
+
+    // sanity checks for ascii alphabet against unexpected locale issues.
+    assertEquals("Aa", StringUtils.camelize("aA"));
+    assertEquals("Bb", StringUtils.camelize("bB"));
+    assertEquals("Cc", StringUtils.camelize("cC"));
+    assertEquals("Dd", StringUtils.camelize("dD"));
+    assertEquals("Ee", StringUtils.camelize("eE"));
+    assertEquals("Ff", StringUtils.camelize("fF"));
+    assertEquals("Gg", StringUtils.camelize("gG"));
+    assertEquals("Hh", StringUtils.camelize("hH"));
+    assertEquals("Ii", StringUtils.camelize("iI"));
+    assertEquals("Jj", StringUtils.camelize("jJ"));
+    assertEquals("Kk", StringUtils.camelize("kK"));
+    assertEquals("Ll", StringUtils.camelize("lL"));
+    assertEquals("Mm", StringUtils.camelize("mM"));
+    assertEquals("Nn", StringUtils.camelize("nN"));
+    assertEquals("Oo", StringUtils.camelize("oO"));
+    assertEquals("Pp", StringUtils.camelize("pP"));
+    assertEquals("Qq", StringUtils.camelize("qQ"));
+    assertEquals("Rr", StringUtils.camelize("rR"));
+    assertEquals("Ss", StringUtils.camelize("sS"));
+    assertEquals("Tt", StringUtils.camelize("tT"));
+    assertEquals("Uu", StringUtils.camelize("uU"));
+    assertEquals("Vv", StringUtils.camelize("vV"));
+    assertEquals("Ww", StringUtils.camelize("wW"));
+    assertEquals("Xx", StringUtils.camelize("xX"));
+    assertEquals("Yy", StringUtils.camelize("yY"));
+    assertEquals("Zz", StringUtils.camelize("zZ"));
+  }
 }



Mime
View raw message