hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject hadoop git commit: YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Date Mon, 24 Apr 2017 19:23:35 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 421e51cb9 -> 49a2f286f


YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos
Szegedi via Haibo Chen)

(cherry picked from commit 8ac50e1322cb3f84bd998635924d85846aa47c94)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a2f286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a2f286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a2f286

Branch: refs/heads/branch-2
Commit: 49a2f286f72bab644fd6aa55b5bc51ae47120e1f
Parents: 421e51c
Author: Haibo Chen <haibochen@cloudera.com>
Authored: Mon Apr 24 11:37:52 2017 -0700
Committer: Haibo Chen <haibochen@cloudera.com>
Committed: Mon Apr 24 11:39:55 2017 -0700

----------------------------------------------------------------------
 .../linux/resources/CGroupsHandlerImpl.java     |  3 +-
 .../util/CgroupsLCEResourcesHandler.java        | 13 ++++++--
 .../linux/resources/TestCGroupsHandlerImpl.java | 32 ++++++++++++++++++++
 .../util/TestCgroupsLCEResourcesHandler.java    | 29 ++++++++++++++++++
 4 files changed, 73 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index d5295c5..0f4c17e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -232,7 +232,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
    * @param entries map of paths to mount options
    * @return the first mount path that has the requested subsystem
    */
-  private static String findControllerInMtab(String controller,
+  @VisibleForTesting
+  static String findControllerInMtab(String controller,
       Map<String, List<String>> entries) {
     for (Map.Entry<String, List<String>> e : entries.entrySet()) {
       if (e.getValue().contains(controller)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index f04fcd2..cb4dcf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -428,11 +428,18 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler
{
     return ret;
   }
 
-  private String findControllerInMtab(String controller,
+  @VisibleForTesting
+  String findControllerInMtab(String controller,
                                       Map<String, List<String>> entries) {
     for (Entry<String, List<String>> e : entries.entrySet()) {
-      if (e.getValue().contains(controller))
-        return e.getKey();
+      if (e.getValue().contains(controller)) {
+        if (new File(e.getKey()).canRead()) {
+          return e.getKey();
+        } else {
+          LOG.warn(String.format(
+              "Skipping inaccessible cgroup mount point %s", e.getKey()));
+        }
+      }
     }
 
     return null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index 4c0829e..804f8e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -40,6 +40,9 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.security.Permission;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
@@ -432,6 +435,35 @@ public class TestCGroupsHandlerImpl {
     }
   }
 
+  @Test
+  public void testSelectCgroup() throws Exception {
+    File cpu = new File(tmpPath, "cpu");
+    File cpuNoExist = new File(tmpPath, "cpuNoExist");
+    File memory = new File(tmpPath, "memory");
+    try {
+      CGroupsHandlerImpl handler = new CGroupsHandlerImpl(
+          conf,
+          privilegedOperationExecutorMock);
+      Map<String, List<String>> cgroups = new LinkedHashMap<>();
+
+      Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+      Assert.assertTrue("temp dir should be created", memory.mkdirs());
+      Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
+
+      cgroups.put(
+          memory.getAbsolutePath(), Collections.singletonList("memory"));
+      cgroups.put(
+          cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
+      cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+      String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
+      Assert.assertEquals("Wrong CPU mount point selected",
+          cpu.getAbsolutePath(), selectedCPU);
+    } finally {
+      FileUtils.deleteQuietly(cpu);
+      FileUtils.deleteQuietly(memory);
+    }
+  }
+
   @After
   public void teardown() {
     FileUtil.fullyDelete(new File(tmpPath));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index 34c9ad1..83c8d5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -33,7 +33,10 @@ import org.junit.Before;
 import org.mockito.Mockito;
 
 import java.io.*;
+import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Scanner;
 import java.util.concurrent.CountDownLatch;
 
@@ -342,4 +345,30 @@ public class TestCgroupsLCEResourcesHandler {
     FileUtils.deleteQuietly(cgroupDir);
   }
 
+  @Test
+  public void testSelectCgroup() {
+    File cpu = new File(cgroupDir, "cpu");
+    File cpuNoExist = new File(cgroupDir, "cpuNoExist");
+    File memory = new File(cgroupDir, "memory");
+    try {
+      CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+      Map<String, List<String>> cgroups = new LinkedHashMap<>();
+
+      Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+      Assert.assertTrue("temp dir should be created", memory.mkdirs());
+      Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
+
+      cgroups.put(
+          memory.getAbsolutePath(), Collections.singletonList("memory"));
+      cgroups.put(
+          cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
+      cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+      String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
+      Assert.assertEquals("Wrong CPU mount point selected",
+          cpu.getAbsolutePath(), selectedCPU);
+    } finally {
+      FileUtils.deleteQuietly(cpu);
+      FileUtils.deleteQuietly(memory);
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message