hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1352389 - in /hadoop/common/branches/branch-1-win: ./ src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/util/ src/winutils/
Date Thu, 21 Jun 2012 01:34:32 GMT
Author: acmurthy
Date: Thu Jun 21 01:34:31 2012
New Revision: 1352389

URL: http://svn.apache.org/viewvc?rev=1352389&view=rev
Log:
MAPREDUCE-4203. Added an implementation of the process tree for Windows. Contributed by Bikas
Saha.

Added:
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsResourceCalculatorPlugin.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsBasedProcessTree.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsResourceCalculatorPlugin.java
    hadoop/common/branches/branch-1-win/src/winutils/systeminfo.c
Modified:
    hadoop/common/branches/branch-1-win/CHANGES.txt
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
    hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorProcessTree.java
    hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
    hadoop/common/branches/branch-1-win/src/winutils/common.h
    hadoop/common/branches/branch-1-win/src/winutils/main.c
    hadoop/common/branches/branch-1-win/src/winutils/task.c
    hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj

Modified: hadoop/common/branches/branch-1-win/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.txt?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.txt Thu Jun 21 01:34:31 2012
@@ -39,6 +39,9 @@ branch-hadoop-1-win - unreleased
 
     HADOOP-8454 Fix the ‘chmod =[perm]’ bug in winutils (Chuan Liu via sanjay)
 
+    MAPREDUCE-4203. Added an implementation of the process tree for Windows.
+    (Bikas Saha via acmurthy) 
+
 Release 1.1.0 - unreleased
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/LinuxResourceCalculatorPlugin.java
Thu Jun 21 01:34:31 2012
@@ -104,8 +104,6 @@ public class LinuxResourceCalculatorPlug
     procfsCpuFile = PROCFS_CPUINFO;
     procfsStatFile = PROCFS_STAT;
     jiffyLengthInMillis = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS;
-    String pid = System.getenv().get("JVM_PID");
-    pTree = new ProcfsBasedProcessTree(pid);
   }
   
   /**
@@ -124,8 +122,6 @@ public class LinuxResourceCalculatorPlug
     this.procfsCpuFile = procfsCpuFile;
     this.procfsStatFile = procfsStatFile;
     this.jiffyLengthInMillis = jiffyLengthInMillis;
-    String pid = System.getenv().get("JVM_PID");
-    pTree = new ProcfsBasedProcessTree(pid);
   }
 
   /**
@@ -396,9 +392,17 @@ public class LinuxResourceCalculatorPlug
     System.out.println("CPU usage % : " + plugin.getCpuUsage());
   }
 
+  /** {@inheritDoc} */
   @Override
   public ProcResourceValues getProcResourceValues() {
-    pTree = pTree.getProcessTree();
+    if(pTree == null) {
+      if(processPid == null) {
+        // process pid not set. try to obtain on our own
+        processPid = System.getenv().get("JVM_PID");
+      }
+      pTree = new ProcfsBasedProcessTree(processPid);
+    }
+    pTree.getProcessTree();
     long cpuTime = pTree.getCumulativeCpuTime();
     long pMem = pTree.getCumulativeRssmem();
     long vMem = pTree.getCumulativeVmem();

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorPlugin.java
Thu Jun 21 01:34:31 2012
@@ -27,6 +27,18 @@ import org.apache.hadoop.util.Reflection
  * 
  */
 public abstract class ResourceCalculatorPlugin extends Configured {
+  
+  protected String processPid = null;
+
+  /**
+   * set the pid of the process for which <code>getProcResourceValues</code>
+   * will be invoked
+   * 
+   * @param pid
+   */
+  public void setProcessPid(String pid) {
+    processPid = pid;
+  }
 
   /**
    * Obtain the total size of the virtual memory present in the system.
@@ -148,6 +160,9 @@ public abstract class ResourceCalculator
       if (osName.startsWith("Linux")) {
         return new LinuxResourceCalculatorPlugin();
       }
+      if (Shell.WINDOWS) {
+        return new WindowsResourceCalculatorPlugin();
+      }
     } catch (SecurityException se) {
       // Failed to get Operating System name.
       return null;

Modified: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorProcessTree.java?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorProcessTree.java
(original)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/ResourceCalculatorProcessTree.java
Thu Jun 21 01:34:31 2012
@@ -112,6 +112,8 @@ public abstract class ResourceCalculator
   public static boolean isAvailable() {
     if(ProcfsBasedProcessTree.isAvailable())
       return true;
+    if(WindowsBasedProcessTree.isAvailable())
+      return true;
     
     return false;
   }
@@ -130,6 +132,8 @@ public abstract class ResourceCalculator
       String pid, Configuration conf) {
     if(ProcfsBasedProcessTree.isAvailable())
       return new ProcfsBasedProcessTree(pid);
+    if(WindowsBasedProcessTree.isAvailable())
+      return new WindowsBasedProcessTree(pid);
 
     // Not supported on this system.
     return null;

Added: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java?rev=1352389&view=auto
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
(added)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsBasedProcessTree.java
Thu Jun 21 01:34:31 2012
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+
+
+public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
+
+  static final Log LOG = LogFactory
+      .getLog(WindowsBasedProcessTree.class);
+  
+  static class ProcessInfo {
+    String pid; // process pid
+    long vmem; // virtual memory
+    long workingSet; // working set, RAM used
+    long cpuTimeMs; // total cpuTime in millisec
+    long cpuTimeMsDelta; // delta of cpuTime since last update
+    int age = 1;
+  }
+  
+  private String taskProcessId = null;
+  private long cpuTimeMs = 0;
+  private Map<String, ProcessInfo> processTree = 
+      new HashMap<String, ProcessInfo>();
+  
+  public static boolean isAvailable() {
+    return Shell.WINDOWS;
+  }
+
+  public WindowsBasedProcessTree(String pid) {
+    taskProcessId = pid;
+  }
+
+  // helper method to override while testing
+  String getAllProcessInfoFromShell() {
+    ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
+        new String[] { Shell.WINUTILS, "task", "processList", taskProcessId });
+    try {
+      shellExecutor.execute();
+      return shellExecutor.getOutput();
+    } catch (IOException e) {
+      LOG.error(StringUtils.stringifyException(e));
+    }
+    return null;
+  }
+
+  /**
+   * Parses string of process info lines into ProcessInfo objects
+   * @param processesInfoStr
+   * @return Map of pid string to ProcessInfo objects
+   */
+  Map<String, ProcessInfo> createProcessInfo(String processesInfoStr) {
+    String[] processesStr = processesInfoStr.split("\r\n");
+    Map<String, ProcessInfo> allProcs = new HashMap<String, ProcessInfo>();
+    final int procInfoSplitCount = 4;
+    for (String processStr : processesStr) {
+      if (processStr != null) {
+        String[] procInfo = processStr.split(",");
+        if (procInfo.length == procInfoSplitCount) {
+          try {
+            ProcessInfo pInfo = new ProcessInfo();
+            pInfo.pid = procInfo[0];
+            pInfo.vmem = Long.parseLong(procInfo[1]);
+            pInfo.workingSet = Long.parseLong(procInfo[2]);
+            pInfo.cpuTimeMs = Long.parseLong(procInfo[3]);
+            allProcs.put(pInfo.pid, pInfo);
+          } catch (NumberFormatException nfe) {
+            LOG.debug("Error parsing procInfo." + nfe);
+          }
+        } else {
+          LOG.debug("Expected split length of proc info to be "
+              + procInfoSplitCount + ". Got " + procInfo.length);
+        }
+      }
+    }
+    return allProcs;
+  }
+  
+  @Override
+  public ResourceCalculatorProcessTree getProcessTree() {
+    String processesInfoStr = getAllProcessInfoFromShell();
+    if (processesInfoStr != null && processesInfoStr.length() > 0) {
+      Map<String, ProcessInfo> allProcessInfo = createProcessInfo(processesInfoStr);
+      
+      for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet()) {
+        String pid = entry.getKey();
+        ProcessInfo pInfo = entry.getValue();
+        ProcessInfo oldInfo = processTree.get(pid);
+        if (oldInfo != null) {
+          // existing process, update age and replace value
+          pInfo.age += oldInfo.age;
+          // calculate the delta since the last refresh. totals are being kept
+          // in the WindowsBasedProcessTree object
+          pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs;
+        } else {
+          // new process. delta cpu == total cpu
+          pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs;
+        }
+      }
+      processTree.clear();
+      processTree = allProcessInfo;
+    }
+    else {
+      processTree.clear();
+    }
+
+    return this;
+  }
+
+  @Override
+  public String getProcessTreeDump() {
+    StringBuilder ret = new StringBuilder();
+    // The header.
+    ret.append(String.format("\t|- PID " + "CPU_TIME(MILLIS) "
+        + "VMEM(BYTES) WORKING_SET(BYTES)\n"));
+    for (ProcessInfo p : processTree.values()) {
+      if (p != null) {
+        ret.append(String.format("\t|- %s %d %d %d\n", p.pid,
+            p.cpuTimeMs, p.vmem, p.workingSet));
+      }
+    }
+    return ret.toString();
+  }
+
+  @Override
+  public long getCumulativeVmem(int olderThanAge) {
+    long total = 0;
+    for (ProcessInfo p : processTree.values()) {
+      if ((p != null) && (p.age > olderThanAge)) {
+        total += p.vmem;
+      }
+    }
+    return total;
+  }
+
+  @Override
+  public long getCumulativeRssmem(int olderThanAge) {
+    long total = 0;
+    for (ProcessInfo p : processTree.values()) {
+      if ((p != null) && (p.age > olderThanAge)) {
+        total += p.workingSet;
+      }
+    }
+    return total;
+  }
+
+  @Override
+  public long getCumulativeCpuTime() {
+    for (ProcessInfo p : processTree.values()) {
+      cpuTimeMs += p.cpuTimeMsDelta;
+    }
+    return cpuTimeMs;
+  }
+
+}

Added: hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsResourceCalculatorPlugin.java?rev=1352389&view=auto
==============================================================================
--- hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsResourceCalculatorPlugin.java
(added)
+++ hadoop/common/branches/branch-1-win/src/core/org/apache/hadoop/util/WindowsResourceCalculatorPlugin.java
Thu Jun 21 01:34:31 2012
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+
+public class WindowsResourceCalculatorPlugin extends ResourceCalculatorPlugin {
+  
+  static final Log LOG = LogFactory
+      .getLog(WindowsResourceCalculatorPlugin.class);
+  
+  long vmemSize;
+  long memSize;
+  long vmemAvailable;
+  long memAvailable;
+  int numProcessors;
+  long cpuFrequencyKhz;
+  long cumulativeCpuTimeMs;
+  float cpuUsage;
+  
+  long lastRefreshTime;
+  private final int refreshIntervalMs = 1000;
+  
+  WindowsBasedProcessTree pTree = null;
+  
+  public WindowsResourceCalculatorPlugin() {
+    lastRefreshTime = 0;
+    reset();
+  }
+  
+  void reset() {
+    vmemSize = -1;
+    memSize = -1;
+    vmemAvailable = -1;
+    memAvailable = -1;
+    numProcessors = -1;
+    cpuFrequencyKhz = -1;
+    cumulativeCpuTimeMs = -1;
+    cpuUsage = -1;
+  }
+
+  String getSystemInfoInfoFromShell() {
+    ShellCommandExecutor shellExecutor = new ShellCommandExecutor(
+        new String[] { Shell.WINUTILS, "systeminfo" });
+    try {
+      shellExecutor.execute();
+      return shellExecutor.getOutput();
+    } catch (IOException e) {
+      LOG.error(StringUtils.stringifyException(e));
+    }
+    return null;
+  }
+  
+  void refreshIfNeeded() {
+    long now = System.currentTimeMillis();
+    if (now - lastRefreshTime > refreshIntervalMs) {
+      long refreshInterval = now - lastRefreshTime;
+      lastRefreshTime = now;
+      long lastCumCpuTimeMs = cumulativeCpuTimeMs;
+      reset();
+      String sysInfoStr = getSystemInfoInfoFromShell();
+      final int sysInfoSplitCount = 7;
+      String[] sysInfo = sysInfoStr.substring(0, sysInfoStr.indexOf("\r\n"))
+          .split(",");
+      if (sysInfo.length == sysInfoSplitCount) {
+        try {
+          vmemSize = Long.parseLong(sysInfo[0]);
+          memSize = Long.parseLong(sysInfo[1]);
+          vmemAvailable = Long.parseLong(sysInfo[2]);
+          memAvailable = Long.parseLong(sysInfo[3]);
+          numProcessors = Integer.parseInt(sysInfo[4]);
+          cpuFrequencyKhz = Long.parseLong(sysInfo[5]);
+          cumulativeCpuTimeMs = Long.parseLong(sysInfo[6]);
+          if (lastCumCpuTimeMs != -1) {
+            cpuUsage = (cumulativeCpuTimeMs - lastCumCpuTimeMs)
+                / (refreshInterval * 1.0f);
+          }
+
+        } catch (NumberFormatException nfe) {
+          LOG.debug("Error parsing sysInfo." + nfe);
+        }
+      } else {
+        LOG.debug("Expected split length of sysInfo to be " + sysInfoSplitCount
+            + ". Got " + sysInfo.length);
+      }
+    }
+  }
+  
+  /** {@inheritDoc} */
+  @Override
+  public long getVirtualMemorySize() {
+    refreshIfNeeded();
+    return vmemSize;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public long getPhysicalMemorySize() {
+    refreshIfNeeded();
+    return memSize;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public long getAvailableVirtualMemorySize() {
+    refreshIfNeeded();
+    return vmemAvailable;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public long getAvailablePhysicalMemorySize() {
+    refreshIfNeeded();
+    return memAvailable;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public int getNumProcessors() {
+    refreshIfNeeded();
+    return numProcessors;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public long getCpuFrequency() {
+    refreshIfNeeded();
+    return -1;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public long getCumulativeCpuTime() {
+    refreshIfNeeded();
+    return cumulativeCpuTimeMs;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public float getCpuUsage() {
+    refreshIfNeeded();
+    return cpuUsage;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public ProcResourceValues getProcResourceValues() {
+    if(pTree == null) {
+      pTree = new WindowsBasedProcessTree(processPid);
+    }
+    pTree.getProcessTree();
+    long cpuTime = pTree.getCumulativeCpuTime();
+    long pMem = pTree.getCumulativeRssmem();
+    long vMem = pTree.getCumulativeVmem();
+    return new ProcResourceValues(cpuTime, pMem, vMem);
+  }
+}

Modified: hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/branch-1-win/src/mapred/org/apache/hadoop/mapred/Task.java Thu
Jun 21 01:34:31 2012
@@ -533,6 +533,7 @@ abstract public class Task implements Wr
             .getResourceCalculatorPlugin(clazz, conf);
     LOG.info(" Using ResourceCalculatorPlugin : " + resourceCalculator);
     if (resourceCalculator != null) {
+      resourceCalculator.setProcessPid(jvmContext.pid);
       initCpuCumulativeTime =
         resourceCalculator.getProcResourceValues().getCumulativeCpuTime();
     }

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
Thu Jun 21 01:34:31 2012
@@ -56,7 +56,7 @@ public class TestTaskTrackerMemoryManage
   private MiniMRCluster miniMRCluster;
 
   private String taskOverLimitPatternString =
-      "TaskTree \\[pid=[0-9]*,tipID=.*\\] is running beyond memory-limits. "
+      "TaskTree \\[pid=.*,tipID=.*\\] is running beyond memory-limits. "
           + "Current usage : [0-9]*bytes. Limit : %sbytes. Killing task.";
 
   private void startCluster(JobConf conf)

Added: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsBasedProcessTree.java?rev=1352389&view=auto
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsBasedProcessTree.java
(added)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsBasedProcessTree.java
Thu Jun 21 01:34:31 2012
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import junit.framework.TestCase;
+
+public class TestWindowsBasedProcessTree extends TestCase {
+  private static final Log LOG = LogFactory
+      .getLog(TestWindowsBasedProcessTree.class);
+  
+  class WindowsBasedProcessTreeTester extends WindowsBasedProcessTree {
+    String infoStr = null;
+    public WindowsBasedProcessTreeTester(String pid) {
+      super(pid);
+    }
+    @Override
+    String getAllProcessInfoFromShell() {
+      return infoStr;
+    }
+  }
+  
+  public void testTree() {
+    if(!WindowsBasedProcessTree.isAvailable()) {
+      LOG.info("WindowsBasedProcessTree not available on this platform. Not testing");
+      return;
+    }
+    
+    WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1");
+    pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n";
+    pTree.getProcessTree();
+    assertTrue(pTree.getCumulativeVmem() == 2048);
+    assertTrue(pTree.getCumulativeVmem(0) == 2048);
+    assertTrue(pTree.getCumulativeRssmem() == 2048);
+    assertTrue(pTree.getCumulativeRssmem(0) == 2048);
+    assertTrue(pTree.getCumulativeCpuTime() == 1000);
+
+    pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n";
+    pTree.getProcessTree();
+    assertTrue(pTree.getCumulativeVmem() == 3072);
+    assertTrue(pTree.getCumulativeVmem(1) == 2048);
+    assertTrue(pTree.getCumulativeRssmem() == 3072);
+    assertTrue(pTree.getCumulativeRssmem(1) == 2048);
+    assertTrue(pTree.getCumulativeCpuTime() == 3000);    
+
+    pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n";
+    pTree.getProcessTree();
+    assertTrue(pTree.getCumulativeVmem() == 2048);
+    assertTrue(pTree.getCumulativeVmem(2) == 2048);
+    assertTrue(pTree.getCumulativeRssmem() == 2048);
+    assertTrue(pTree.getCumulativeRssmem(2) == 2048);
+    assertTrue(pTree.getCumulativeCpuTime() == 4000);    
+  }
+}

Added: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsResourceCalculatorPlugin.java?rev=1352389&view=auto
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsResourceCalculatorPlugin.java
(added)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/util/TestWindowsResourceCalculatorPlugin.java
Thu Jun 21 01:34:31 2012
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import junit.framework.TestCase;
+
+public class TestWindowsResourceCalculatorPlugin extends TestCase {
+  
+  
+  class WindowsResourceCalculatorPluginTester extends WindowsResourceCalculatorPlugin {
+    private String infoStr = null;
+    @Override
+    String getSystemInfoInfoFromShell() {
+      return infoStr;
+    }    
+  }
+  
+  public void testParseSystemInfoString() {
+    WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
+    // info str derived from windows shell command has \r\n termination
+    tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n";
+    // call a method to refresh values
+    tester.getAvailablePhysicalMemorySize();
+    // verify information has been refreshed
+    assertTrue(tester.vmemSize == 17177038848L);
+    assertTrue(tester.memSize == 8589467648L);
+    assertTrue(tester.vmemAvailable == 15232745472L);
+    assertTrue(tester.memAvailable == 6400417792L);
+    assertTrue(tester.numProcessors == 1);
+    assertTrue(tester.cpuFrequencyKhz == 2805000L);
+    assertTrue(tester.cumulativeCpuTimeMs == 6261812L);
+    assertTrue(tester.cpuUsage == -1);
+  }
+  
+  public void testRefreshAndCpuUsage() throws InterruptedException {
+    WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester();
+    // info str derived from windows shell command has \r\n termination
+    tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n";
+    tester.getAvailablePhysicalMemorySize();
+    // verify information has been refreshed
+    assertTrue(tester.memAvailable == 6400417792L);
+    assertTrue(tester.cpuUsage == -1);
+    
+    tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6261812\r\n";
+    tester.getAvailablePhysicalMemorySize();
+    // verify information has not been refreshed
+    assertTrue(tester.memAvailable == 6400417792L);
+    assertTrue(tester.cpuUsage == -1);
+    
+    Thread.sleep(1500);
+    tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6286812\r\n";
+    tester.getAvailablePhysicalMemorySize();
+    // verify information has been refreshed
+    assertTrue(tester.memAvailable == 5400417792L);
+    assertTrue(tester.cpuUsage >= 0.1);
+  }
+
+}

Modified: hadoop/common/branches/branch-1-win/src/winutils/common.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/winutils/common.h?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/winutils/common.h (original)
+++ hadoop/common/branches/branch-1-win/src/winutils/common.h Thu Jun 21 01:34:31 2012
@@ -86,6 +86,9 @@ void HardlinkUsage();
 int Task(int argc, wchar_t *argv[]);
 void TaskUsage();
 
+int SystemInfo();
+void SystemInfoUsage();
+
 DWORD GetFileInformationByName(__in LPCWSTR pathName,
   __out LPBY_HANDLE_FILE_INFORMATION lpFileInformation);
 

Modified: hadoop/common/branches/branch-1-win/src/winutils/main.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/winutils/main.c?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/winutils/main.c (original)
+++ hadoop/common/branches/branch-1-win/src/winutils/main.c Thu Jun 21 01:34:31 2012
@@ -55,6 +55,10 @@ int wmain(int argc, wchar_t* argv[])
   {
     return Task(argc - 1, argv + 1);
   }
+  else if (wcscmp(L"systeminfo", cmd) == 0)
+  {
+    return SystemInfo();
+  }
   else
   {
     Usage(argv[0]);
@@ -71,27 +75,32 @@ static void Usage(LPCWSTR program)
 Provide basic command line utilities for Hadoop on Windows.\n\n\
 The available commands and their usages are:\n\n", program);
 
-  fwprintf(stdout, L"%-10s%s\n\n", L"ls", L"List file information.");
+  fwprintf(stdout, L"%-15s%s\n\n", L"ls", L"List file information.");
   LsUsage(L"ls");
   fwprintf(stdout, L"\n\n");
 
-  fwprintf(stdout, L"%-10s%s\n\n", L"chmod", L"Change file mode bits.");
+  fwprintf(stdout, L"%-15s%s\n\n", L"chmod", L"Change file mode bits.");
   ChmodUsage(L"chmod");
   fwprintf(stdout, L"\n\n");
     
-  fwprintf(stdout, L"%-10s%s\n\n", L"chown", L"Change file owner.");
+  fwprintf(stdout, L"%-15s%s\n\n", L"chown", L"Change file owner.");
   ChownUsage(L"chown");
   fwprintf(stdout, L"\n\n");
   
-  fwprintf(stdout, L"%-10s%s\n\n", L"groups", L"List user groups.");
+  fwprintf(stdout, L"%-15s%s\n\n", L"groups", L"List user groups.");
   GroupsUsage(L"groups");
   fwprintf(stdout, L"\n\n");
 
-  fwprintf(stdout, L"%-10s%s\n\n", L"hardlink", L"Hard link operations.");
+  fwprintf(stdout, L"%-15s%s\n\n", L"hardlink", L"Hard link operations.");
   HardlinkUsage();
   fwprintf(stdout, L"\n\n");
 
-  fwprintf(stdout, L"%-10s%s\n\n", L"task", L"Task operations.");
+  fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations.");
   TaskUsage();
   fwprintf(stdout, L"\n\n");
+
+  fwprintf(stdout, L"%-15s%s\n\n", L"systeminfo", L"System information.");
+  SystemInfoUsage();
+
+  fwprintf(stdout, L"\n\n");
 }
\ No newline at end of file

Added: hadoop/common/branches/branch-1-win/src/winutils/systeminfo.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/winutils/systeminfo.c?rev=1352389&view=auto
==============================================================================
--- hadoop/common/branches/branch-1-win/src/winutils/systeminfo.c (added)
+++ hadoop/common/branches/branch-1-win/src/winutils/systeminfo.c Thu Jun 21 01:34:31 2012
@@ -0,0 +1,120 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with this
+* work for additional information regarding copyright ownership. The ASF
+* licenses this file to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* 
+* http://www.apache.org/licenses/LICENSE-2.0
+* 
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*/
+
+#include "common.h"
+#include <psapi.h>
+#include <PowrProf.h>
+
+#define PSAPI_VERSION 1
+#pragma comment(lib, "psapi.lib")
+#pragma comment(lib, "Powrprof.lib")
+
+typedef struct _PROCESSOR_POWER_INFORMATION {
+   ULONG  Number;
+   ULONG  MaxMhz;
+   ULONG  CurrentMhz;
+   ULONG  MhzLimit;
+   ULONG  MaxIdleState;
+   ULONG  CurrentIdleState;
+} PROCESSOR_POWER_INFORMATION, *PPROCESSOR_POWER_INFORMATION;
+
+//----------------------------------------------------------------------------
+// Function: SystemInfo
+//
+// Description:
+// Returns the resource information about the machine 
+//
+// Returns:
+// EXIT_SUCCESS: On success
+// EXIT_FAILURE: otherwise
+int SystemInfo() 
+{
+  size_t vmemSize, vmemFree, memSize, memFree;
+  PERFORMANCE_INFORMATION memInfo;
+  SYSTEM_INFO sysInfo;
+  FILETIME idleTimeFt, kernelTimeFt, userTimeFt;
+  ULARGE_INTEGER idleTime, kernelTime, userTime;
+  ULONGLONG cpuTimeMs;
+  size_t size;
+  LPBYTE pBuffer;
+  PPROCESSOR_POWER_INFORMATION ppi;
+  long cpuFrequencyKhz;
+  NTSTATUS status;
+
+  ZeroMemory(&memInfo, sizeof(PERFORMANCE_INFORMATION));
+  memInfo.cb = sizeof(PERFORMANCE_INFORMATION);
+  if(!GetPerformanceInfo(&memInfo, sizeof(PERFORMANCE_INFORMATION)))
+  {
+    ReportErrorCode(L"GetPerformanceInfo", GetLastError());
+    return EXIT_FAILURE;
+  }
+  vmemSize = memInfo.CommitLimit*memInfo.PageSize;
+  vmemFree = vmemSize - memInfo.CommitTotal*memInfo.PageSize;
+  memSize = memInfo.PhysicalTotal*memInfo.PageSize;
+  memFree = memInfo.PhysicalAvailable*memInfo.PageSize;
+
+  GetSystemInfo(&sysInfo);
+
+  if(!GetSystemTimes(&idleTimeFt, &kernelTimeFt, &userTimeFt))
+  {
+    ReportErrorCode(L"GetSystemTimes", GetLastError());
+    return EXIT_FAILURE;
+  }
+  idleTime.HighPart = idleTimeFt.dwHighDateTime;
+  idleTime.LowPart = idleTimeFt.dwLowDateTime;
+  kernelTime.HighPart = kernelTimeFt.dwHighDateTime;
+  kernelTime.LowPart = kernelTimeFt.dwLowDateTime;
+  userTime.HighPart = userTimeFt.dwHighDateTime;
+  userTime.LowPart = userTimeFt.dwLowDateTime;
+
+  cpuTimeMs = (kernelTime.QuadPart - idleTime.QuadPart + userTime.QuadPart)/10000;
+
+  // allocate buffer to get info for each processor
+  size = sysInfo.dwNumberOfProcessors * sizeof(PROCESSOR_POWER_INFORMATION);
+  pBuffer = (BYTE*) LocalAlloc(LPTR, size);
+  if(!pBuffer)
+  {
+    ReportErrorCode(L"LocalAlloc", GetLastError());
+    return EXIT_FAILURE;
+  }
+  status = CallNtPowerInformation(ProcessorInformation, NULL, 0, pBuffer, (long)size);
+  if(0 != status)
+  {
+    fwprintf_s(stderr, L"Error in CallNtPowerInformation. Err:%d\n", status);
+    LocalFree(pBuffer);
+    return EXIT_FAILURE;
+  }
+  ppi = (PPROCESSOR_POWER_INFORMATION)pBuffer;
+  cpuFrequencyKhz = ppi->MaxMhz*1000;
+  LocalFree(pBuffer);
+
+  fwprintf_s(stdout, L"%Iu,%Iu,%Iu,%Iu,%Iu,%Iu,%Iu\n", vmemSize, memSize, vmemFree, memFree,
sysInfo.dwNumberOfProcessors, cpuFrequencyKhz, cpuTimeMs);
+
+  return EXIT_SUCCESS;
+}
+
+void SystemInfoUsage()
+{
+    fwprintf(stdout, L"\
+    Usage: systeminfo\n\
+    Prints machine information on stdout\n\
+    Comma separated list of the following values.\n\
+    VirtualMemorySize(bytes),PhysicalMemorySize(bytes),\n\
+    FreeVirtualMemory(bytes),FreePhysicalMemory(bytes),\n\
+    NumberOfProcessors,CpuFrequency(Khz),\n\
+    CpuTime(MilliSec,Kernel+User)\n");
+}
\ No newline at end of file

Modified: hadoop/common/branches/branch-1-win/src/winutils/task.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/winutils/task.c?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/winutils/task.c (original)
+++ hadoop/common/branches/branch-1-win/src/winutils/task.c Thu Jun 21 01:34:31 2012
@@ -17,15 +17,20 @@
 
 #include "common.h"
 #include <errno.h>
+#include <psapi.h>
 
-// List of different hardlink related command line options supported by
+#define PSAPI_VERSION 1
+#pragma comment(lib, "psapi.lib")
+
+// List of different task related command line options supported by
 // winutils.
 typedef enum TaskCommandOptionType
 {
   TaskInvalid,
   TaskCreate,
   TaskIsAlive,
-  TaskKill
+  TaskKill,
+  TaskProcessList
 } TaskCommandOption;
 
 //----------------------------------------------------------------------------
@@ -60,6 +65,11 @@ static BOOL ParseCommandLine(__in int ar
       *command = TaskKill;
       return TRUE;
     }
+    if (wcscmp(argv[1], L"processList") == 0)
+    {
+      *command = TaskProcessList;
+      return TRUE;
+    }
   }
 
   if (argc == 4) {
@@ -247,7 +257,7 @@ DWORD killTask(_TCHAR* jobObjName)
     return err;
   }
 
-  if(TerminateJobObject(jobObject, -1) == 0)
+  if(TerminateJobObject(jobObject, 1) == 0)
   {
     return GetLastError();
   }
@@ -257,6 +267,85 @@ DWORD killTask(_TCHAR* jobObjName)
 }
 
 //----------------------------------------------------------------------------
+// Function: printTaskProcessList
+//
+// Description:
+// Prints resource usage of all processes in the task jobobject
+//
+// Returns:
+// ERROR_SUCCESS: On success
+// GetLastError: otherwise
+DWORD printTaskProcessList(const _TCHAR* jobObjName)
+{
+  DWORD i;
+  PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
+  int numProcs = 100;
+  HANDLE jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName);
+  if(jobObject == NULL)
+  {
+    DWORD err = GetLastError();
+    return err;
+  }
+
+  procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST)
+ numProcs*32);
+  if (!procList)
+  {
+    DWORD err = GetLastError();
+    CloseHandle(jobObject);
+    return err;
+  }
+  while(QueryInformationJobObject(jobObject, JobObjectBasicProcessIdList, procList, sizeof(JOBOBJECT_BASIC_PROCESS_ID_LIST)+numProcs*32,
NULL) == 0)
+  {
+    DWORD err = GetLastError();
+    if(err != ERROR_MORE_DATA) 
+    {
+      CloseHandle(jobObject);
+      LocalFree(procList);
+      return err;
+    }
+    numProcs = procList->NumberOfAssignedProcesses;
+    LocalFree(procList);
+    procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST)
+ numProcs*32);
+    if (!procList)
+    {
+      DWORD err = GetLastError();
+      CloseHandle(jobObject);
+      return err;
+    }
+  }
+
+  for(i=0; i<procList->NumberOfProcessIdsInList; ++i)
+  {
+    HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION, FALSE, (DWORD)procList->ProcessIdList[i]
);
+    if( hProcess != NULL )
+    {
+      PROCESS_MEMORY_COUNTERS_EX pmc;
+      if ( GetProcessMemoryInfo( hProcess, (PPROCESS_MEMORY_COUNTERS)&pmc, sizeof(pmc))
)
+      {
+        FILETIME create, exit, kernel, user;
+        if( GetProcessTimes( hProcess, &create, &exit, &kernel, &user) )
+        {
+          ULARGE_INTEGER kernelTime, userTime;
+          ULONGLONG cpuTimeMs;
+          kernelTime.HighPart = kernel.dwHighDateTime;
+          kernelTime.LowPart = kernel.dwLowDateTime;
+          userTime.HighPart = user.dwHighDateTime;
+          userTime.LowPart = user.dwLowDateTime;
+          cpuTimeMs = (kernelTime.QuadPart+userTime.QuadPart)/10000;
+          _ftprintf_s(stdout, TEXT("%u,%Iu,%Iu,%Iu\n"), procList->ProcessIdList[i], pmc.PrivateUsage,
pmc.WorkingSetSize, cpuTimeMs);
+        }
+      }
+      CloseHandle( hProcess );
+    }
+  }
+
+  LocalFree(procList);
+  CloseHandle(jobObject);
+
+  return ERROR_SUCCESS;
+}
+
+//----------------------------------------------------------------------------
 // Function: Task
 //
 // Description:
@@ -317,6 +406,16 @@ int Task(int argc, wchar_t *argv[])
       ReportErrorCode(L"killTask", dwErrorCode);
       goto TaskExit;
     }
+  } else if (command == TaskProcessList)
+  {
+    // Check if task jobobject
+    //
+    dwErrorCode = printTaskProcessList(argv[2]);
+    if (dwErrorCode != ERROR_SUCCESS)
+    {
+      ReportErrorCode(L"printTaskProcessList", dwErrorCode);
+      goto TaskExit;
+    }
   } else
   {
     // Should not happen
@@ -337,7 +436,13 @@ void TaskUsage()
     Usage: task create [TASKNAME] [COMMAND_LINE] |\n\
           task isAlive [TASKNAME] |\n\
           task kill [TASKNAME]\n\
+          task processList [TASKNAME]\n\
     Creates a new task jobobject with taskname\n\
     Checks if task jobobject is alive\n\
-    Kills task jobobject\n");
+    Kills task jobobject\n\
+    Prints to stdout a list of processes in the task\n\
+    along with their resource usage. One process per line\n\
+    and comma separated info per process\n\
+    ProcessId,VirtualMemoryCommitted(bytes),\n\
+    WorkingSetSize(bytes),CpuTime(Millisec,Kernel+User)\n");
 }
\ No newline at end of file

Modified: hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj?rev=1352389&r1=1352388&r2=1352389&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj (original)
+++ hadoop/common/branches/branch-1-win/src/winutils/winutils.vcxproj Thu Jun 21 01:34:31
2012
@@ -135,6 +135,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="systeminfo.c" />
     <ClCompile Include="chmod.c" />
     <ClCompile Include="chown.c" />
     <ClCompile Include="common.c" />
@@ -150,4 +151,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
\ No newline at end of file



Mime
View raw message