hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [08/49] hadoop git commit: YARN-8880. Add configurations for pluggable plugin framework. Contributed by Zhankun Tang.
Date Tue, 13 Nov 2018 07:50:35 GMT
YARN-8880. Add configurations for pluggable plugin framework. Contributed by Zhankun Tang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c72d7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c72d7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c72d7b

Branch: refs/heads/HDFS-13891
Commit: f8c72d7b3acca8285bbc3024f491c4586805be1e
Parents: c96cbe8
Author: Weiwei Yang <wwei@apache.org>
Authored: Thu Nov 8 12:23:00 2018 +0800
Committer: Weiwei Yang <wwei@apache.org>
Committed: Thu Nov 8 12:23:00 2018 +0800

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  24 ++++-
 .../src/main/resources/yarn-default.xml         |  19 ++++
 .../resourceplugin/ResourcePluginManager.java   |  34 +++++-
 .../TestResourcePluginManager.java              | 107 +++++++++++++++++--
 4 files changed, 170 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ce38d27..e88d594 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1606,6 +1606,28 @@ public class YarnConfiguration extends Configuration {
       NM_PREFIX + "resource-plugins";
 
   /**
+   * This setting controls if pluggable device plugin framework is enabled.
+   * */
+  @Private
+  public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+      NM_PREFIX + "pluggable-device-framework.enabled";
+
+  /**
+   * The pluggable device plugin framework is disabled by default
+   * */
+  @Private
+  public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED =
+      false;
+
+  /**
+   * This setting contains vendor plugin class names for
+   * device plugin framework to load. Split by comma
+   * */
+  @Private
+  public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES =
+      NM_PREFIX + "pluggable-device-framework.device-classes";
+
+  /**
    * Prefix for gpu configurations. Work in progress: This configuration
    * parameter may be changed/removed in the future.
    */
@@ -1647,7 +1669,7 @@ public class YarnConfiguration extends Configuration {
       NVIDIA_DOCKER_V1;
 
   /**
-   * This setting controls end point of nvidia-docker-v1 plugin
+   * This setting controls end point of nvidia-docker-v1 plugin.
    */
   @Private
   public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1360e73..f5493bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3772,6 +3772,25 @@
 
   <property>
     <description>
+      This setting controls if pluggable device framework is enabled.
+      Disabled by default
+    </description>
+    <name>yarn.nodemanager.pluggable-device-framework.enabled</name>
+    <value>false</value>
+  </property>
+
+  <property>
+    <description>
+      Configure vendor device plugin class name here. Comma separated.
+      The class must be found in CLASSPATH. The pluggable device framework will
+      load these classes.
+    </description>
+    <name>yarn.nodemanager.pluggable-device-framework.device-classes</name>
+    <value></value>
+  </property>
+
+  <property>
+    <description>
       When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified,
       YARN NodeManager needs to run GPU discovery binary (now only support
       nvidia-smi) to get GPU-related information.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
index f28aad2..e930d96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin;
@@ -52,11 +53,10 @@ public class ResourcePluginManager {
   public synchronized void initialize(Context context)
       throws YarnException {
     Configuration conf = context.getConf();
-    String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
+    Map<String, ResourcePlugin> pluginMap = new HashMap<>();
 
+    String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS);
     if (plugins != null) {
-      Map<String, ResourcePlugin> pluginMap = new HashMap<>();
-
       // Initialize each plugins
       for (String resourceName : plugins) {
         resourceName = resourceName.trim();
@@ -92,8 +92,34 @@ public class ResourcePluginManager {
         plugin.initialize(context);
         pluginMap.put(resourceName, plugin);
       }
+    }
+    // Try to load pluggable device plugins
+    boolean puggableDeviceFrameworkEnabled = conf.getBoolean(
+        YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+        YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+
+    if (puggableDeviceFrameworkEnabled) {
+      initializePluggableDevicePlugins(context, conf, pluginMap);
+    } else {
+      LOG.info("The pluggable device framework is not enabled."
+              + " If you want, please set true to {}",
+          YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED);
+    }
+    configuredPlugins = Collections.unmodifiableMap(pluginMap);
+  }
 
-      configuredPlugins = Collections.unmodifiableMap(pluginMap);
+  public void initializePluggableDevicePlugins(Context context,
+      Configuration configuration,
+      Map<String, ResourcePlugin> pluginMap)
+      throws YarnRuntimeException {
+    LOG.info("The pluggable device framework enabled," +
+        "trying to load the vendor plugins");
+
+    String[] pluginClassNames = configuration.getStrings(
+        YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
+    if (null == pluginClassNames) {
+      throw new YarnRuntimeException("Null value found in configuration: "
+          + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
index 6ed7c56..eb8bf54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -41,12 +42,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.HashMap;
@@ -58,10 +57,18 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.spy;
 
 public class TestResourcePluginManager extends NodeManagerTestBase {
   private NodeManager nm;
 
+  private YarnConfiguration conf;
+
+  @Before
+  public void setup() throws Exception {
+    this.conf = createNMConfig();
+  }
+
   ResourcePluginManager stubResourcePluginmanager() {
     // Stub ResourcePluginManager
     final ResourcePluginManager rpm = mock(ResourcePluginManager.class);
@@ -183,7 +190,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
     final ResourcePluginManager rpm = stubResourcePluginmanager();
     nm = new MyMockNM(rpm);
 
-    YarnConfiguration conf = createNMConfig();
     nm.init(conf);
     verify(rpm, times(1)).initialize(
         any(Context.class));
@@ -198,7 +204,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
 
     nm = new MyMockNM(rpm);
 
-    YarnConfiguration conf = createNMConfig();
     nm.init(conf);
     nm.start();
 
@@ -238,15 +243,14 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
       }
 
       @Override
-      protected ContainerExecutor createContainerExecutor(Configuration conf) {
+      protected ContainerExecutor createContainerExecutor(
+          Configuration configuration) {
         ((NMContext)this.getNMContext()).setResourcePluginManager(rpm);
-        lce.setConf(conf);
+        lce.setConf(configuration);
         return lce;
       }
     };
 
-    YarnConfiguration conf = createNMConfig();
-
     nm.init(conf);
     nm.start();
 
@@ -264,4 +268,89 @@ public class TestResourcePluginManager extends NodeManagerTestBase {
     }
     Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded);
   }
+
+  // Disabled pluggable framework in configuration.
+  // We use spy object of real rpm to verify "initializePluggableDevicePlugins"
+  // because use mock rpm will not working
+  @Test(timeout = 30000)
+  public void testInitializationWithPluggableDeviceFrameworkDisabled()
+      throws Exception {
+    ResourcePluginManager rpm = new ResourcePluginManager();
+
+    ResourcePluginManager rpmSpy = spy(rpm);
+    nm = new MyMockNM(rpmSpy);
+
+    conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+        false);
+    nm.init(conf);
+    nm.start();
+    verify(rpmSpy, times(1)).initialize(
+        any(Context.class));
+    verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+        any(Context.class), any(Configuration.class), any(Map.class));
+  }
+
+  // No related configuration set.
+  @Test(timeout = 30000)
+  public void testInitializationWithPluggableDeviceFrameworkDisabled2()
+      throws Exception {
+    ResourcePluginManager rpm = new ResourcePluginManager();
+
+    ResourcePluginManager rpmSpy = spy(rpm);
+    nm = new MyMockNM(rpmSpy);
+
+    nm.init(conf);
+    nm.start();
+    verify(rpmSpy, times(1)).initialize(
+        any(Context.class));
+    verify(rpmSpy, times(0)).initializePluggableDevicePlugins(
+        any(Context.class), any(Configuration.class), any(Map.class));
+  }
+
+  // Enable framework and configure pluggable device classes
+  @Test(timeout = 30000)
+  public void testInitializationWithPluggableDeviceFrameworkEnabled()
+      throws Exception {
+    ResourcePluginManager rpm = new ResourcePluginManager();
+
+    ResourcePluginManager rpmSpy = spy(rpm);
+    nm = new MyMockNM(rpmSpy);
+
+    conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+        true);
+    conf.setStrings(
+        YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES,
+        "com.cmp1.hdw1plugin");
+    nm.init(conf);
+    nm.start();
+    verify(rpmSpy, times(1)).initialize(
+        any(Context.class));
+    verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+        any(Context.class), any(Configuration.class), any(Map.class));
+  }
+
+  // Enable pluggable framework, but leave device classes un-configured
+  // initializePluggableDevicePlugins invoked but it should throw an exception
+  @Test(timeout = 30000)
+  public void testInitializationWithPluggableDeviceFrameworkEnabled2() {
+    ResourcePluginManager rpm = new ResourcePluginManager();
+
+    ResourcePluginManager rpmSpy = spy(rpm);
+    nm = new MyMockNM(rpmSpy);
+    Boolean fail = false;
+    try {
+      conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED,
+          true);
+
+      nm.init(conf);
+      nm.start();
+    } catch (YarnRuntimeException e) {
+      fail = true;
+    } catch (Exception e) {
+
+    }
+    verify(rpmSpy, times(1)).initializePluggableDevicePlugins(
+        any(Context.class), any(Configuration.class), any(Map.class));
+    Assert.assertTrue(fail);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message