Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4212718C73 for ; Tue, 22 Dec 2015 23:51:19 +0000 (UTC) Received: (qmail 22466 invoked by uid 500); 22 Dec 2015 23:51:03 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 21915 invoked by uid 500); 22 Dec 2015 23:51:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 20661 invoked by uid 99); 22 Dec 2015 23:51:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 22 Dec 2015 23:51:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B07C2E08DD; Tue, 22 Dec 2015 23:51:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Date: Tue, 22 Dec 2015 23:51:47 -0000 Message-Id: <51f91ecfeaca41bab0ce8564f54c8171@git.apache.org> In-Reply-To: <827b4547bc0e4a2685c6d00bbe27ad32@git.apache.org> References: <827b4547bc0e4a2685c6d00bbe27ad32@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [48/50] [abbrv] hadoop git commit: YARN-3458. CPU resource monitoring in Windows. Contributed by Inigo Goiri. YARN-3458. CPU resource monitoring in Windows. Contributed by Inigo Goiri. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/114b5909 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/114b5909 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/114b5909 Branch: refs/heads/yarn-2877 Commit: 114b59095540bb80db5153c816f9d285e4029031 Parents: a0de702 Author: cnauroth Authored: Mon Dec 21 16:59:09 2015 -0800 Committer: cnauroth Committed: Mon Dec 21 16:59:09 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 + .../yarn/util/WindowsBasedProcessTree.java | 51 ++++++++++++++++++-- .../yarn/util/TestWindowsBasedProcessTree.java | 25 ++++++++-- 3 files changed, 68 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/114b5909/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 10f54e6..c306c04 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -283,6 +283,8 @@ Release 2.8.0 - UNRELEASED YARN-3226. UI changes for decommissioning node. (Sunil G via junping_du) + YARN-3458. CPU resource monitoring in Windows. (Inigo Goiri via cnauroth) + IMPROVEMENTS YARN-644. Basic null check is not performed on passed in arguments before http://git-wip-us.apache.org/repos/asf/hadoop/blob/114b5909/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java index 1bf25a5..30add01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java @@ -19,12 +19,14 @@ package org.apache.hadoop.yarn.util; import java.io.IOException; +import java.math.BigInteger; import java.util.HashMap; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.util.CpuTimeTracker; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; @@ -48,7 +50,12 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree { private long cpuTimeMs = UNAVAILABLE; private Map processTree = new HashMap(); - + + /** Track CPU utilization. */ + private final CpuTimeTracker cpuTimeTracker; + /** Clock to account for CPU utilization. */ + private Clock clock; + public static boolean isAvailable() { if (Shell.WINDOWS) { if (!Shell.hasWinutilsPath()) { @@ -71,9 +78,25 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree { return false; } - public WindowsBasedProcessTree(String pid) { + /** + * Create a monitor for a Windows process tree. + * @param pid Identifier of the job object. + */ + public WindowsBasedProcessTree(final String pid) { + this(pid, new SystemClock()); + } + + /** + * Create a monitor for a Windows process tree. + * @param pid Identifier of the job object. + * @param pClock Clock to keep track of time for CPU utilization. + */ + public WindowsBasedProcessTree(final String pid, final Clock pClock) { super(pid); - taskProcessId = pid; + this.taskProcessId = pid; + this.clock = pClock; + // Instead of jiffies, Windows uses milliseconds directly; 1ms = 1 jiffy + this.cpuTimeTracker = new CpuTimeTracker(1L); } // helper method to override while testing @@ -213,7 +236,7 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree { } return total; } - + @Override @SuppressWarnings("deprecation") public long getCumulativeRssmem(int olderThanAge) { @@ -231,9 +254,27 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree { return cpuTimeMs; } + /** + * Get the number of used ms for all the processes under the monitored job + * object. + * @return Total consumed milliseconds by all processes in the job object. + */ + private BigInteger getTotalProcessMs() { + long totalMs = 0; + for (ProcessInfo p : processTree.values()) { + if (p != null) { + totalMs += p.cpuTimeMs; + } + } + return BigInteger.valueOf(totalMs); + } + @Override public float getCpuUsagePercent() { - return UNAVAILABLE; + BigInteger processTotalMs = getTotalProcessMs(); + cpuTimeTracker.updateElapsedJiffies(processTotalMs, clock.getTime()); + + return cpuTimeTracker.getCpuTrackerUsagePercent(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/114b5909/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java index 80c5b02..c28915c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java @@ -21,8 +21,9 @@ package org.apache.hadoop.yarn.util; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.Shell; - +import org.junit.Assert; import org.junit.Test; + import static org.junit.Assert.assertTrue; public class TestWindowsBasedProcessTree { @@ -31,8 +32,9 @@ public class TestWindowsBasedProcessTree { class WindowsBasedProcessTreeTester extends WindowsBasedProcessTree { String infoStr = null; - public WindowsBasedProcessTreeTester(String pid) { - super(pid); + + public WindowsBasedProcessTreeTester(String pid, Clock clock) { + super(pid, clock); } @Override String getAllProcessInfoFromShell() { @@ -49,8 +51,11 @@ public class TestWindowsBasedProcessTree { } assertTrue("WindowsBasedProcessTree should be available on Windows", WindowsBasedProcessTree.isAvailable()); - - WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1"); + ControlledClock testClock = new ControlledClock(new SystemClock()); + long elapsedTimeBetweenUpdatesMsec = 0; + testClock.setTime(elapsedTimeBetweenUpdatesMsec); + + WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1", testClock); pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n"; pTree.updateProcessTree(); assertTrue(pTree.getVirtualMemorySize() == 2048); @@ -63,8 +68,11 @@ public class TestWindowsBasedProcessTree { assertTrue(pTree.getRssMemorySize(0) == 2048); assertTrue(pTree.getCumulativeRssmem(0) == 2048); assertTrue(pTree.getCumulativeCpuTime() == 1000); + assertTrue(pTree.getCpuUsagePercent() == ResourceCalculatorProcessTree.UNAVAILABLE); pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n"; + elapsedTimeBetweenUpdatesMsec = 1000; + testClock.setTime(elapsedTimeBetweenUpdatesMsec); pTree.updateProcessTree(); assertTrue(pTree.getVirtualMemorySize() == 3072); assertTrue(pTree.getCumulativeVmem() == 3072); @@ -75,8 +83,13 @@ public class TestWindowsBasedProcessTree { assertTrue(pTree.getRssMemorySize(1) == 2048); assertTrue(pTree.getCumulativeRssmem(1) == 2048); assertTrue(pTree.getCumulativeCpuTime() == 3000); + assertTrue(pTree.getCpuUsagePercent() == 200); + Assert.assertEquals("Percent CPU time is not correct", + pTree.getCpuUsagePercent(), 200, 0.01); pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n"; + elapsedTimeBetweenUpdatesMsec = 2000; + testClock.setTime(elapsedTimeBetweenUpdatesMsec); pTree.updateProcessTree(); assertTrue(pTree.getVirtualMemorySize() == 2048); assertTrue(pTree.getCumulativeVmem() == 2048); @@ -87,5 +100,7 @@ public class TestWindowsBasedProcessTree { assertTrue(pTree.getRssMemorySize(2) == 2048); assertTrue(pTree.getCumulativeRssmem(2) == 2048); assertTrue(pTree.getCumulativeCpuTime() == 4000); + Assert.assertEquals("Percent CPU time is not correct", + pTree.getCpuUsagePercent(), 0, 0.01); } }