hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l..@apache.org
Subject svn commit: r1424242 - in /hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn: hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/ hadoop-yarn-server/hadoop-yarn-serve...
Date Thu, 20 Dec 2012 00:19:41 GMT
Author: llu
Date: Thu Dec 20 00:19:40 2012
New Revision: 1424242

URL: http://svn.apache.org/viewvc?rev=1424242&view=rev
Log:
YARN-223. Update process tree instead of getting new process trees. (Radim Kolar via llu)

Modified:
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
    hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
Thu Dec 20 00:19:40 2012
@@ -91,7 +91,6 @@ public class LinuxResourceCalculatorPlug
   private float cpuUsage = UNAVAILABLE;
   private long sampleTime = UNAVAILABLE;
   private long lastSampleTime = UNAVAILABLE;
-  private ResourceCalculatorProcessTree pTree = null;
 
   boolean readMemInfoFile = false;
   boolean readCpuInfoFile = false;
@@ -109,8 +108,6 @@ public class LinuxResourceCalculatorPlug
     procfsCpuFile = PROCFS_CPUINFO;
     procfsStatFile = PROCFS_STAT;
     jiffyLengthInMillis = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS;
-    String pid = System.getenv().get("JVM_PID");
-    pTree = new ProcfsBasedProcessTree(pid);
   }
 
   /**
@@ -129,8 +126,6 @@ public class LinuxResourceCalculatorPlug
     this.procfsCpuFile = procfsCpuFile;
     this.procfsStatFile = procfsStatFile;
     this.jiffyLengthInMillis = jiffyLengthInMillis;
-    String pid = System.getenv().get("JVM_PID");
-    pTree = new ProcfsBasedProcessTree(pid);
   }
 
   /**
@@ -400,13 +395,4 @@ public class LinuxResourceCalculatorPlug
     }
     System.out.println("CPU usage % : " + plugin.getCpuUsage());
   }
-
-  @Override
-  public ProcResourceValues getProcResourceValues() {
-    pTree = pTree.getProcessTree();
-    long cpuTime = pTree.getCumulativeCpuTime();
-    long pMem = pTree.getCumulativeRssmem();
-    long vMem = pTree.getCumulativeVmem();
-    return new ProcResourceValues(cpuTime, pMem, vMem);
-  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
Thu Dec 20 00:19:40 2012
@@ -140,13 +140,12 @@ public class ProcfsBasedProcessTree exte
   }
 
   /**
-   * Get the process-tree with latest state. If the root-process is not alive,
-   * an empty tree will be returned.
+   * Update process-tree with latest state. If the root-process is not alive,
+   * tree will be empty.
    *
-   * @return the process-tree with latest state.
    */
   @Override
-  public ResourceCalculatorProcessTree getProcessTree() {
+  public void updateProcessTree() {
     if (!pid.equals(deadPid)) {
       // Get the list of processes
       List<String> processList = getProcessList();
@@ -172,7 +171,7 @@ public class ProcfsBasedProcessTree exte
       }
 
       if (me == null) {
-        return this;
+        return;
       }
 
       // Add each process to its parent.
@@ -214,7 +213,6 @@ public class ProcfsBasedProcessTree exte
         LOG.debug(this.toString());
       }
     }
-    return this;
   }
 
   /** Verify that the given process id is same as its process group id.

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
Thu Dec 20 00:19:40 2012
@@ -91,48 +91,6 @@ public abstract class ResourceCalculator
   public abstract float getCpuUsage();
 
   /**
-   * Obtain resource status used by current process tree.
-   */
-  @InterfaceAudience.Private
-  @InterfaceStability.Unstable
-  public abstract ProcResourceValues getProcResourceValues();
-
-  public static class ProcResourceValues {
-    private final long cumulativeCpuTime;
-    private final long physicalMemorySize;
-    private final long virtualMemorySize;
-    public ProcResourceValues(long cumulativeCpuTime, long physicalMemorySize,
-                              long virtualMemorySize) {
-      this.cumulativeCpuTime = cumulativeCpuTime;
-      this.physicalMemorySize = physicalMemorySize;
-      this.virtualMemorySize = virtualMemorySize;
-    }
-    /**
-     * Obtain the physical memory size used by current process tree.
-     * @return physical memory size in bytes.
-     */
-    public long getPhysicalMemorySize() {
-      return physicalMemorySize;
-    }
-
-    /**
-     * Obtain the virtual memory size used by a current process tree.
-     * @return virtual memory size in bytes.
-     */
-    public long getVirtualMemorySize() {
-      return virtualMemorySize;
-    }
-
-    /**
-     * Obtain the cumulative CPU time used by a current process tree.
-     * @return cumulative CPU time in milliseconds
-     */
-    public long getCumulativeCpuTime() {
-      return cumulativeCpuTime;
-    }
-  }
-
-  /**
    * Create the ResourceCalculatorPlugin from the class name and configure it. If
    * class name is null, this method will try and return a memory calculator
    * plugin available for this system.

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java
Thu Dec 20 00:19:40 2012
@@ -43,16 +43,14 @@ public abstract class ResourceCalculator
   }
 
   /**
-   * Get the process-tree with latest state. If the root-process is not alive,
-   * an empty tree will be returned.
+   * Update the process-tree with latest state.
    *
    * Each call to this function should increment the age of the running
    * processes that already exist in the process tree. Age is used other API's
    * of the interface.
    *
-   * @return the process-tree with latest state.
    */
-  public abstract ResourceCalculatorProcessTree getProcessTree();
+  public abstract void updateProcessTree();
 
   /**
    * Get a dump of the process-tree.

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java
Thu Dec 20 00:19:40 2012
@@ -161,7 +161,7 @@ public class TestProcfsBasedProcessTree 
     String pid = getRogueTaskPID();
     LOG.info("Root process pid: " + pid);
     ProcfsBasedProcessTree p = createProcessTree(pid);
-    p.getProcessTree(); // initialize
+    p.updateProcessTree(); // initialize
     LOG.info("ProcessTree: " + p.toString());
 
     File leaf = new File(lowestDescendant);
@@ -174,7 +174,7 @@ public class TestProcfsBasedProcessTree 
       }
     }
 
-    p.getProcessTree(); // reconstruct
+    p.updateProcessTree(); // reconstruct
     LOG.info("ProcessTree: " + p.toString());
 
     // Get the process-tree dump
@@ -213,7 +213,7 @@ public class TestProcfsBasedProcessTree 
     }
 
     // ProcessTree is gone now. Any further calls should be sane.
-    p.getProcessTree();
+    p.updateProcessTree();
     Assert.assertFalse("ProcessTree must have been gone", isAlive(pid));
     Assert.assertTrue("Cumulative vmem for the gone-process is "
         + p.getCumulativeVmem() + " . It should be zero.", p
@@ -358,7 +358,7 @@ public class TestProcfsBasedProcessTree 
       ProcfsBasedProcessTree processTree =
           createProcessTree("100", procfsRootDir.getAbsolutePath());
       // build the process tree.
-      processTree.getProcessTree();
+      processTree.updateProcessTree();
 
       // verify cumulative memory
       Assert.assertEquals("Cumulative virtual memory does not match", 600000L,
@@ -384,7 +384,7 @@ public class TestProcfsBasedProcessTree 
       writeStatFiles(procfsRootDir, pids, procInfos);
 
       // build the process tree.
-      processTree.getProcessTree();
+      processTree.updateProcessTree();
 
       // verify cumulative cpu time again
       cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
@@ -431,7 +431,7 @@ public class TestProcfsBasedProcessTree 
       ProcfsBasedProcessTree processTree =
           createProcessTree("100", procfsRootDir.getAbsolutePath());
       // build the process tree.
-      processTree.getProcessTree();
+      processTree.updateProcessTree();
 
       // verify cumulative memory
       Assert.assertEquals("Cumulative memory does not match",
@@ -447,7 +447,7 @@ public class TestProcfsBasedProcessTree 
       writeStatFiles(procfsRootDir, newPids, newProcInfos);
 
       // check memory includes the new process.
-      processTree.getProcessTree();
+      processTree.updateProcessTree();
       Assert.assertEquals("Cumulative vmem does not include new process",
                    1200000L, processTree.getCumulativeVmem());
       long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
@@ -473,7 +473,7 @@ public class TestProcfsBasedProcessTree 
       writeStatFiles(procfsRootDir, newPids, newProcInfos);
 
       // refresh process tree
-      processTree.getProcessTree();
+      processTree.updateProcessTree();
 
       // processes older than 2 iterations should be same as before.
       Assert.assertEquals("Cumulative vmem shouldn't have included new processes",
@@ -577,7 +577,7 @@ public class TestProcfsBasedProcessTree 
       ProcfsBasedProcessTree processTree = createProcessTree(
           "100", procfsRootDir.getAbsolutePath());
       // build the process tree.
-      processTree.getProcessTree();
+      processTree.updateProcessTree();
 
       // Get the process-tree dump
       String processTreeDump = processTree.getProcessTreeDump();

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestResourceCalculatorProcessTree.java
Thu Dec 20 00:19:40 2012
@@ -34,8 +34,7 @@ public class TestResourceCalculatorProce
       super(pid);
     }
 
-    public ResourceCalculatorProcessTree getProcessTree() {
-      return this;
+    public void updateProcessTree() {
     }
 
     public String getProcessTreeDump() {

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
Thu Dec 20 00:19:40 2012
@@ -396,9 +396,7 @@ public class ContainersMonitorImpl exten
             LOG.debug("Constructing ProcessTree for : PID = " + pId
                 + " ContainerId = " + containerId);
             ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
-            pTree = pTree.getProcessTree(); // get the updated process-tree
-            ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
-                                          // updated state
+            pTree.updateProcessTree();    // update process-tree
             long currentVmemUsage = pTree.getCumulativeVmem();
             long currentPmemUsage = pTree.getCumulativeRssmem();
             // as processes begin with an age 1, we want to see if there

Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1424242&r1=1424241&r2=1424242&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
Thu Dec 20 00:19:40 2012
@@ -134,7 +134,7 @@ public class TestContainersMonitor exten
       ProcfsBasedProcessTree pTree = new ProcfsBasedProcessTree(
                                           "100",
                                           procfsRootDir.getAbsolutePath());
-      pTree.getProcessTree();
+      pTree.updateProcessTree();
       assertTrue("tree rooted at 100 should be over limit " +
                     "after first iteration.",
                   test.isProcessTreeOverLimit(pTree, "dummyId", limit));
@@ -142,13 +142,13 @@ public class TestContainersMonitor exten
       // the tree rooted at 200 is initially below limit.
       pTree = new ProcfsBasedProcessTree("200",
                                           procfsRootDir.getAbsolutePath());
-      pTree.getProcessTree();
+      pTree.updateProcessTree();
       assertFalse("tree rooted at 200 shouldn't be over limit " +
                     "after one iteration.",
                   test.isProcessTreeOverLimit(pTree, "dummyId", limit));
       // second iteration - now the tree has been over limit twice,
       // hence it should be declared over limit.
-      pTree.getProcessTree();
+      pTree.updateProcessTree();
       assertTrue(
           "tree rooted at 200 should be over limit after 2 iterations",
                   test.isProcessTreeOverLimit(pTree, "dummyId", limit));
@@ -156,12 +156,12 @@ public class TestContainersMonitor exten
       // the tree rooted at 600 is never over limit.
       pTree = new ProcfsBasedProcessTree("600",
                                             procfsRootDir.getAbsolutePath());
-      pTree.getProcessTree();
+      pTree.updateProcessTree();
       assertFalse("tree rooted at 600 should never be over limit.",
                     test.isProcessTreeOverLimit(pTree, "dummyId", limit));
 
       // another iteration does not make any difference.
-      pTree.getProcessTree();
+      pTree.updateProcessTree();
       assertFalse("tree rooted at 600 should never be over limit.",
                     test.isProcessTreeOverLimit(pTree, "dummyId", limit));
     } finally {



Mime
View raw message