hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1357475 - in /hadoop/common/branches/branch-1-win: ./ src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/ src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/
Date Thu, 05 Jul 2012 07:05:25 GMT
Author: acmurthy
Date: Thu Jul  5 07:05:24 2012
New Revision: 1357475

URL: http://svn.apache.org/viewvc?rev=1357475&view=rev
Log:
MAPREDUCE-4369. Fix streaming test failures caused by WindowsResourceCalculatorPlugin. Contributed
by Bikas Saha. 

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.txt
    hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Thu Jul  5 07:05:24 2012
@@ -53,6 +53,9 @@ branch-hadoop-1-win - unreleased
     MAPREDUCE-4332 Fix command-length abort issues on Windows. (Ivan Mitic via
     acmurthy)
 
+    MAPREDUCE-4369. Fix streaming test failures caused by
+    WindowsResourceCalculatorPlugin. (Bikas Saha via acmurthy) 
+
 Release 1.1.0 - unreleased
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
(original)
+++ hadoop/common/branches/branch-1-win/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
Thu Jul  5 07:05:24 2012
@@ -185,6 +185,10 @@ class LoadJob extends GridmixJob {
       ResourceCalculatorPlugin plugin = 
         ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
       
+      if(plugin != null) {
+        plugin.setProcessPid(System.getenv("JVM_PID"));
+      }
+      
       // set the other parameters
       this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
       progress = new BoostingProgress(context);

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
Thu Jul  5 07:05:24 2012
@@ -396,10 +396,6 @@ public class LinuxResourceCalculatorPlug
   @Override
   public ProcResourceValues getProcResourceValues() {
     if(pTree == null) {
-      if(processPid == null) {
-        // process pid not set. try to obtain on our own
-        processPid = System.getenv().get("JVM_PID");
-      }
       pTree = new ProcfsBasedProcessTree(processPid);
     }
     pTree.getProcessTree();

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
Thu Jul  5 07:05:24 2012
@@ -100,6 +100,8 @@ public abstract class ResourceCalculator
 
   /**
    * Obtain resource status used by current process tree.
+   * Must call {@link #setProcessPid(String)} to set the process id before 
+   * calling this method
    */
   public abstract ProcResourceValues getProcResourceValues();
 

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
Thu Jul  5 07:05:24 2012
@@ -101,30 +101,33 @@ public class WindowsBasedProcessTree ext
   
   @Override
   public ResourceCalculatorProcessTree getProcessTree() {
-    String processesInfoStr = getAllProcessInfoFromShell();
-    if (processesInfoStr != null && processesInfoStr.length() > 0) {
-      Map<String, ProcessInfo> allProcessInfo = createProcessInfo(processesInfoStr);
-      
-      for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet()) {
-        String pid = entry.getKey();
-        ProcessInfo pInfo = entry.getValue();
-        ProcessInfo oldInfo = processTree.get(pid);
-        if (oldInfo != null) {
-          // existing process, update age and replace value
-          pInfo.age += oldInfo.age;
-          // calculate the delta since the last refresh. totals are being kept
-          // in the WindowsBasedProcessTree object
-          pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs;
-        } else {
-          // new process. delta cpu == total cpu
-          pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs;
+    if(taskProcessId != null) {
+      // taskProcessId can be null in some tests
+      String processesInfoStr = getAllProcessInfoFromShell();
+      if (processesInfoStr != null && processesInfoStr.length() > 0) {
+        Map<String, ProcessInfo> allProcessInfo = createProcessInfo(processesInfoStr);
+
+        for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet()) {
+          String pid = entry.getKey();
+          ProcessInfo pInfo = entry.getValue();
+          ProcessInfo oldInfo = processTree.get(pid);
+          if (oldInfo != null) {
+            // existing process, update age and replace value
+            pInfo.age += oldInfo.age;
+            // calculate the delta since the last refresh. totals are being kept
+            // in the WindowsBasedProcessTree object
+            pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs;
+          } else {
+            // new process. delta cpu == total cpu
+            pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs;
+          }
         }
+        processTree.clear();
+        processTree = allProcessInfo;
+      } else {
+        // clearing process tree to mimic semantics of existing Procfs impl
+        processTree.clear();
       }
-      processTree.clear();
-      processTree = allProcessInfo;
-    }
-    else {
-      processTree.clear();
     }
 
     return this;

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java?rev=1357475&r1=1357474&r2=1357475&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java Thu
Jul  5 07:05:24 2012
@@ -533,9 +533,14 @@ abstract public class Task implements Wr
             .getResourceCalculatorPlugin(clazz, conf);
     LOG.info(" Using ResourceCalculatorPlugin : " + resourceCalculator);
     if (resourceCalculator != null) {
-      resourceCalculator.setProcessPid(jvmContext.pid);
-      initCpuCumulativeTime =
-        resourceCalculator.getProcResourceValues().getCumulativeCpuTime();
+      if (jvmContext != null) {
+        resourceCalculator.setProcessPid(jvmContext.pid);
+        initCpuCumulativeTime = resourceCalculator.getProcResourceValues()
+            .getCumulativeCpuTime();
+      } else {
+        // jvmContext is not set in local job mode
+        resourceCalculator = null;
+      }
     }
   }
   



Mime
View raw message