hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xkro...@apache.org
Subject [31/50] [abbrv] hadoop git commit: YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan.
Date Tue, 26 Jun 2018 05:33:09 GMT
YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64321286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64321286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64321286

Branch: refs/heads/HDFS-12943
Commit: 6432128622d64f3f9dd638b9c254c77cdf5408aa
Parents: 30728ac
Author: Eric E Payne <ericp@oath.com>
Authored: Fri Jun 22 17:15:29 2018 +0000
Committer: Eric E Payne <ericp@oath.com>
Committed: Fri Jun 22 17:15:29 2018 +0000

----------------------------------------------------------------------
 .../org/apache/hadoop/util/SysInfoLinux.java    | 19 ++++++-
 .../apache/hadoop/util/TestSysInfoLinux.java    | 60 ++++++++++++++++++++
 2 files changed, 77 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 7fd1990..2c2aca3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -217,6 +217,21 @@ public class SysInfoLinux extends SysInfo {
   }
 
   /**
+   *
+   * Wrapper for Long.parseLong() that returns zero if the value is
+   * invalid. Under some circumstances, swapFree in /proc/meminfo can
+   * go negative, reported as a very large decimal value.
+   */
+  private long safeParseLong(String strVal) {
+    long parsedVal;
+    try {
+      parsedVal = Long.parseLong(strVal);
+    } catch (NumberFormatException nfe) {
+      parsedVal = 0;
+    }
+    return parsedVal;
+  }
+  /**
    * Read /proc/meminfo, parse and compute memory information.
    * @param readAgain if false, read only on the first time
    */
@@ -252,9 +267,9 @@ public class SysInfoLinux extends SysInfo {
           } else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
             swapSize = Long.parseLong(mat.group(2));
           } else if (mat.group(1).equals(MEMFREE_STRING)) {
-            ramSizeFree = Long.parseLong(mat.group(2));
+            ramSizeFree = safeParseLong(mat.group(2));
           } else if (mat.group(1).equals(SWAPFREE_STRING)) {
-            swapSizeFree = Long.parseLong(mat.group(2));
+            swapSizeFree = safeParseLong(mat.group(2));
           } else if (mat.group(1).equals(INACTIVE_STRING)) {
             inactiveSize = Long.parseLong(mat.group(2));
           } else if (mat.group(1).equals(INACTIVEFILE_STRING)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index a646a41..0ae5d3c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -161,6 +161,36 @@ public class TestSysInfoLinux {
     "DirectMap2M:     2027520 kB\n" +
     "DirectMap1G:    132120576 kB\n";
 
+  static final String MEMINFO_FORMAT3 =
+    "MemTotal:      %d kB\n" +
+    "MemFree:         %s kB\n" +
+    "Buffers:        138244 kB\n" +
+    "Cached:         947780 kB\n" +
+    "SwapCached:     142880 kB\n" +
+    "Active:        3229888 kB\n" +
+    "Inactive:       %d kB\n" +
+    "SwapTotal:     %d kB\n" +
+    "SwapFree:      %s kB\n" +
+    "Dirty:          122012 kB\n" +
+    "Writeback:           0 kB\n" +
+    "AnonPages:     2710792 kB\n" +
+    "Mapped:          24740 kB\n" +
+    "Slab:           132528 kB\n" +
+    "SReclaimable:   105096 kB\n" +
+    "SUnreclaim:      27432 kB\n" +
+    "PageTables:      11448 kB\n" +
+    "NFS_Unstable:        0 kB\n" +
+    "Bounce:              0 kB\n" +
+    "CommitLimit:   4125904 kB\n" +
+    "Committed_AS:  4143556 kB\n" +
+    "VmallocTotal: 34359738367 kB\n" +
+    "VmallocUsed:      1632 kB\n" +
+    "VmallocChunk: 34359736375 kB\n" +
+    "HugePages_Total:     %d\n" +
+    "HugePages_Free:      0\n" +
+    "HugePages_Rsvd:      0\n" +
+    "Hugepagesize:     2048 kB";
+
   static final String CPUINFO_FORMAT =
     "processor : %s\n" +
     "vendor_id : AuthenticAMD\n" +
@@ -384,6 +414,36 @@ public class TestSysInfoLinux {
                           (nrHugePages * 2048) + swapTotal));
   }
 
+  /**
+   * Test parsing /proc/meminfo
+   * @throws IOException
+   */
+  @Test
+  public void parsingProcMemFileWithBadValues() throws IOException {
+    long memTotal = 4058864L;
+    long memFree = 0L;  // bad value should return 0
+    long inactive = 567732L;
+    long swapTotal = 2096472L;
+    long swapFree = 0L; // bad value should return 0
+    int nrHugePages = 10;
+    String badFreeValue = "18446744073709551596";
+    File tempFile = new File(FAKE_MEMFILE);
+    tempFile.deleteOnExit();
+    FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+    fWriter.write(String.format(MEMINFO_FORMAT3,
+        memTotal, badFreeValue, inactive, swapTotal, badFreeValue, nrHugePages));
+
+    fWriter.close();
+    assertEquals(plugin.getAvailablePhysicalMemorySize(),
+        1024L * (memFree + inactive));
+    assertEquals(plugin.getAvailableVirtualMemorySize(),
+        1024L * (memFree + inactive + swapFree));
+    assertEquals(plugin.getPhysicalMemorySize(),
+        1024L * (memTotal - (nrHugePages * 2048)));
+    assertEquals(plugin.getVirtualMemorySize(),
+        1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
+  }
+
   @Test
   public void testCoreCounts() throws IOException {
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message