Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8DEAF200AE2 for ; Fri, 13 May 2016 01:01:57 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 8C7BC160A1B; Thu, 12 May 2016 23:01:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AC096160A10 for ; Fri, 13 May 2016 01:01:56 +0200 (CEST) Received: (qmail 99724 invoked by uid 500); 12 May 2016 23:01:54 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 99480 invoked by uid 99); 12 May 2016 23:01:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 May 2016 23:01:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 00D61E0B4E; Thu, 12 May 2016 23:01:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aw@apache.org To: common-commits@hadoop.apache.org Date: Thu, 12 May 2016 23:01:58 -0000 Message-Id: <6fa2a268a5754a81b4f90e6d557efc69@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [06/50] [abbrv] hadoop git commit: YARN-4768. getAvailablePhysicalMemorySize can be inaccurate on linux. Contributed by Nathan Roberts. archived-at: Thu, 12 May 2016 23:01:57 -0000 YARN-4768. getAvailablePhysicalMemorySize can be inaccurate on linux. Contributed by Nathan Roberts. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b1c1cb0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b1c1cb0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b1c1cb0 Branch: refs/heads/HADOOP-12930 Commit: 6b1c1cb01cbf979f46cd3ea9308b7745c5595b4f Parents: 025219b Author: Eric Payne Authored: Tue May 10 21:28:02 2016 +0000 Committer: Eric Payne Committed: Tue May 10 21:28:02 2016 +0000 ---------------------------------------------------------------------- .../org/apache/hadoop/util/SysInfoLinux.java | 39 +++++++-- .../apache/hadoop/util/TestSysInfoLinux.java | 92 +++++++++++++++++++- 2 files changed, 119 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b1c1cb0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java index 2852bea..bba1631 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java @@ -53,7 +53,7 @@ public class SysInfoLinux extends SysInfo { */ private static final String PROCFS_MEMFILE = "/proc/meminfo"; private static final Pattern PROCFS_MEMFILE_FORMAT = - Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB"); + Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?"); // We need the values for the following keys in meminfo private static final String MEMTOTAL_STRING = "MemTotal"; @@ -61,6 +61,12 @@ public class SysInfoLinux extends SysInfo { private static final String MEMFREE_STRING = "MemFree"; private static final String SWAPFREE_STRING = "SwapFree"; private static final String INACTIVE_STRING = "Inactive"; + private static final String INACTIVEFILE_STRING = "Inactive(file)"; + private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted"; + private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total"; + private static final String HUGEPAGESIZE_STRING = "Hugepagesize"; + + /** * Patterns for parsing /proc/cpuinfo. @@ -122,7 +128,13 @@ public class SysInfoLinux extends SysInfo { private long swapSize = 0; private long ramSizeFree = 0; // free ram space on the machine (kB) private long swapSizeFree = 0; // free swap space on the machine (kB) - private long inactiveSize = 0; // inactive cache memory (kB) + private long inactiveSize = 0; // inactive memory (kB) + private long inactiveFileSize = -1; // inactive cache memory, -1 if not there + private long hardwareCorruptSize = 0; // RAM corrupt and not available + private long hugePagesTotal = 0; // # of hugepages reserved + private long hugePageSize = 0; // # size of each hugepage + + /* number of logical processors on the system. */ private int numProcessors = 0; /* number of physical cores on the system. */ @@ -245,6 +257,14 @@ public class SysInfoLinux extends SysInfo { swapSizeFree = Long.parseLong(mat.group(2)); } else if (mat.group(1).equals(INACTIVE_STRING)) { inactiveSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(INACTIVEFILE_STRING)) { + inactiveFileSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) { + hardwareCorruptSize = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) { + hugePagesTotal = Long.parseLong(mat.group(2)); + } else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) { + hugePageSize = Long.parseLong(mat.group(2)); } } str = in.readLine(); @@ -554,28 +574,31 @@ public class SysInfoLinux extends SysInfo { @Override public long getPhysicalMemorySize() { readProcMemInfoFile(); - return ramSize * 1024; + return (ramSize + - hardwareCorruptSize + - (hugePagesTotal * hugePageSize)) * 1024; } /** {@inheritDoc} */ @Override public long getVirtualMemorySize() { - readProcMemInfoFile(); - return (ramSize + swapSize) * 1024; + return getPhysicalMemorySize() + (swapSize * 1024); } /** {@inheritDoc} */ @Override public long getAvailablePhysicalMemorySize() { readProcMemInfoFile(true); - return (ramSizeFree + inactiveSize) * 1024; + long inactive = inactiveFileSize != -1 + ? inactiveFileSize + : inactiveSize; + return (ramSizeFree + inactive) * 1024; } /** {@inheritDoc} */ @Override public long getAvailableVirtualMemorySize() { - readProcMemInfoFile(true); - return (ramSizeFree + swapSizeFree + inactiveSize) * 1024; + return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b1c1cb0/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java index 8a6df0c..a646a41 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java @@ -29,6 +29,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * A JUnit test to test {@link SysInfoLinux} @@ -110,11 +111,56 @@ public class TestSysInfoLinux { "VmallocTotal: 34359738367 kB\n" + "VmallocUsed: 1632 kB\n" + "VmallocChunk: 34359736375 kB\n" + - "HugePages_Total: 0\n" + + "HugePages_Total: %d\n" + "HugePages_Free: 0\n" + "HugePages_Rsvd: 0\n" + "Hugepagesize: 2048 kB"; + static final String MEMINFO_FORMAT_2 = + "MemTotal: %d kB\n" + + "MemFree: %d kB\n" + + "Buffers: 129976 kB\n" + + "Cached: 32317676 kB\n" + + "SwapCached: 0 kB\n" + + "Active: 88938588 kB\n" + + "Inactive: %d kB\n" + + "Active(anon): 77502200 kB\n" + + "Inactive(anon): 6385336 kB\n" + + "Active(file): 11436388 kB\n" + + "Inactive(file): %d kB\n" + + "Unevictable: 0 kB\n" + + "Mlocked: 0 kB\n" + + "SwapTotal: %d kB\n" + + "SwapFree: %d kB\n" + + "Dirty: 575864 kB\n" + + "Writeback: 16 kB\n" + + "AnonPages: 83886180 kB\n" + + "Mapped: 108640 kB\n" + + "Shmem: 1880 kB\n" + + "Slab: 2413448 kB\n" + + "SReclaimable: 2194488 kB\n" + + "SUnreclaim: 218960 kB\n" + + "KernelStack: 31496 kB\n" + + "PageTables: 195176 kB\n" + + "NFS_Unstable: 0 kB\n" + + "Bounce: 0 kB\n" + + "WritebackTmp: 0 kB\n" + + "CommitLimit: 97683468 kB\n" + + "Committed_AS: 94553560 kB\n" + + "VmallocTotal: 34359738367 kB\n" + + "VmallocUsed: 498580 kB\n" + + "VmallocChunk: 34256922296 kB\n" + + "HardwareCorrupted: %d kB\n" + + "AnonHugePages: 0 kB\n" + + "HugePages_Total: %d\n" + + "HugePages_Free: 0\n" + + "HugePages_Rsvd: 0\n" + + "HugePages_Surp: 0\n" + + "Hugepagesize: 2048 kB\n" + + "DirectMap4k: 4096 kB\n" + + "DirectMap2M: 2027520 kB\n" + + "DirectMap1G: 132120576 kB\n"; + static final String CPUINFO_FORMAT = "processor : %s\n" + "vendor_id : AuthenticAMD\n" + @@ -285,19 +331,57 @@ public class TestSysInfoLinux { long inactive = 567732L; long swapTotal = 2096472L; long swapFree = 1818480L; + int nrHugePages = 10; File tempFile = new File(FAKE_MEMFILE); tempFile.deleteOnExit(); FileWriter fWriter = new FileWriter(FAKE_MEMFILE); fWriter.write(String.format(MEMINFO_FORMAT, - memTotal, memFree, inactive, swapTotal, swapFree)); + memTotal, memFree, inactive, swapTotal, swapFree, nrHugePages)); fWriter.close(); assertEquals(plugin.getAvailablePhysicalMemorySize(), 1024L * (memFree + inactive)); assertEquals(plugin.getAvailableVirtualMemorySize(), 1024L * (memFree + inactive + swapFree)); - assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal); - assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal)); + assertEquals(plugin.getPhysicalMemorySize(), + 1024L * (memTotal - (nrHugePages * 2048))); + assertEquals(plugin.getVirtualMemorySize(), + 1024L * (memTotal - (nrHugePages * 2048) + swapTotal)); + } + + /** + * Test parsing /proc/meminfo with Inactive(file) present + * @throws IOException + */ + @Test + public void parsingProcMemFile2() throws IOException { + long memTotal = 131403836L; + long memFree = 11257036L; + long inactive = 27396032L; + long inactiveFile = 21010696L; + long swapTotal = 31981552L; + long swapFree = 1818480L; + long hardwareCorrupt = 31960904L; + int nrHugePages = 10; + File tempFile = new File(FAKE_MEMFILE); + tempFile.deleteOnExit(); + FileWriter fWriter = new FileWriter(FAKE_MEMFILE); + fWriter.write(String.format(MEMINFO_FORMAT_2, + memTotal, memFree, inactive, inactiveFile, swapTotal, swapFree, + hardwareCorrupt, nrHugePages)); + + fWriter.close(); + assertEquals(plugin.getAvailablePhysicalMemorySize(), + 1024L * (memFree + inactiveFile)); + assertFalse(plugin.getAvailablePhysicalMemorySize() == + 1024L * (memFree + inactive)); + assertEquals(plugin.getAvailableVirtualMemorySize(), + 1024L * (memFree + inactiveFile + swapFree)); + assertEquals(plugin.getPhysicalMemorySize(), + 1024L * (memTotal - hardwareCorrupt - (nrHugePages * 2048))); + assertEquals(plugin.getVirtualMemorySize(), + 1024L * (memTotal - hardwareCorrupt - + (nrHugePages * 2048) + swapTotal)); } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org