Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 0CF7B200C06 for ; Fri, 27 Jan 2017 19:41:46 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 0B7A4160B40; Fri, 27 Jan 2017 18:41:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 84269160B5B for ; Fri, 27 Jan 2017 19:41:44 +0100 (CET) Received: (qmail 17482 invoked by uid 500); 27 Jan 2017 18:40:07 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 15530 invoked by uid 99); 27 Jan 2017 18:40:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 27 Jan 2017 18:40:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0CE24DFBDB; Fri, 27 Jan 2017 18:40:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Fri, 27 Jan 2017 18:40:47 -0000 Message-Id: <675055cd04bd45b3a9c64a60ae347fe5@git.apache.org> In-Reply-To: <7fccdec439f94cbd8d0afe3fe9340207@git.apache.org> References: <7fccdec439f94cbd8d0afe3fe9340207@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [42/50] [abbrv] hadoop git commit: YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev. archived-at: Fri, 27 Jan 2017 18:41:46 -0000 YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d702592 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d702592 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d702592 Branch: refs/heads/YARN-3926 Commit: 1d702592516b43e1bbed4fdaf9d481b853d4eff0 Parents: 6ed968a Author: Varun Vasudev Authored: Sat Jun 11 14:33:46 2016 +0530 Committer: Varun Vasudev Committed: Fri Jan 27 23:23:36 2017 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/api/records/Resource.java | 3 +- .../hadoop/yarn/conf/YarnConfiguration.java | 19 ++- .../FileSystemBasedConfigurationProvider.java | 3 +- .../hadoop/yarn/LocalConfigurationProvider.java | 3 +- .../api/records/impl/pb/ResourcePBImpl.java | 53 +++--- .../yarn/util/resource/ResourceUtils.java | 168 +++++++++++++++---- .../yarn/util/resource/TestResourceUtils.java | 29 +++- .../resource-types/node-resources-1.xml | 29 ++++ .../resource-types/node-resources-2.xml | 39 +++++ .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../util/NodeManagerHardwareUtils.java | 52 ++++++ .../resourcemanager/ResourceTrackerService.java | 9 +- 12 files changed, 342 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java index 3efccce..3880bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java @@ -272,7 +272,8 @@ public abstract class Resource implements Comparable { continue; } if (entry.getKey().equals(ResourceInformation.VCORES.getName()) - && entry.getValue().getUnits().equals("")) { + && entry.getValue().getUnits() + .equals(ResourceInformation.VCORES.getUnits())) { continue; } sb.append(", ").append(entry.getKey()).append(": ") http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e411659..7959d31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -63,6 +63,10 @@ public class YarnConfiguration extends Configuration { "resource-types.xml"; @Private + public static final String NODE_RESOURCES_CONFIGURATION_FILE = + "node-resources.xml"; + + @Private public static final List RM_CONFIGURATION_FILES = Collections.unmodifiableList(Arrays.asList( RESOURCE_TYPES_CONFIGURATION_FILE, @@ -72,6 +76,16 @@ public class YarnConfiguration extends Configuration { YARN_SITE_CONFIGURATION_FILE, CORE_SITE_CONFIGURATION_FILE)); + @Private + public static final List NM_CONFIGURATION_FILES = + Collections.unmodifiableList(Arrays.asList( + NODE_RESOURCES_CONFIGURATION_FILE, + DR_CONFIGURATION_FILE, + CS_CONFIGURATION_FILE, + HADOOP_POLICY_CONFIGURATION_FILE, + YARN_SITE_CONFIGURATION_FILE, + CORE_SITE_CONFIGURATION_FILE)); + @Evolving public static final int APPLICATION_MAX_TAGS = 10; @@ -100,12 +114,15 @@ public class YarnConfiguration extends Configuration { public static final String YARN_PREFIX = "yarn."; ///////////////////////////// - // Scheduler resource types configs + // Resource types configs //////////////////////////// public static final String RESOURCE_TYPES = YarnConfiguration.YARN_PREFIX + "resource-types"; + public static final String NM_RESOURCES_PREFIX = + YarnConfiguration.NM_PREFIX + "resource-type."; + /** Delay before deleting resource to ease debugging of NM issues */ public static final String DEBUG_NM_DELETE_DELAY_SEC = YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index cef03b9..73c4990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -51,7 +51,8 @@ public class FileSystemBasedConfigurationProvider "Illegal argument! The parameter should not be null or empty"); } Path filePath; - if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name) || + YarnConfiguration.NM_CONFIGURATION_FILES.contains(name)) { filePath = new Path(this.configDir, name); if (!fs.exists(filePath)) { LOG.info(filePath + " not found"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java index cfa194f..0cdbd15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java @@ -39,7 +39,8 @@ public class LocalConfigurationProvider extends ConfigurationProvider { if (name == null || name.isEmpty()) { throw new YarnException( "Illegal argument! The parameter should not be null or empty"); - } else if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { + } else if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name) || + YarnConfiguration.NM_CONFIGURATION_FILES.contains(name)) { return bootstrapConf.getConfResourceAsInputStream(name); } return new FileInputStream(name); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 6ff091c..8cf7291 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -18,7 +18,8 @@ package org.apache.hadoop.yarn.api.records.impl.pb; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; @@ -38,6 +39,8 @@ import java.util.*; @Unstable public class ResourcePBImpl extends Resource { + private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class); + ResourceProto proto = ResourceProto.getDefaultInstance(); ResourceProto.Builder builder = null; boolean viaProto = false; @@ -78,10 +81,12 @@ public class ResourcePBImpl extends Resource { @Override public long getMemorySize() { // memory should always be present - initResourcesMap(); + initResources(); ResourceInformation ri = this.getResourceInformation(ResourceInformation.MEMORY_MB.getName()); - return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue()); + return UnitsConversionUtil + .convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(), + ri.getValue()); } @Override @@ -101,21 +106,16 @@ public class ResourcePBImpl extends Resource { @Override public int getVirtualCores() { // vcores should always be present - initResourcesMap(); + initResources(); return this.getResourceValue(ResourceInformation.VCORES.getName()) .intValue(); } @Override public void setVirtualCores(int vCores) { - try { - setResourceValue(ResourceInformation.VCORES.getName(), - Long.valueOf(vCores)); - } catch (ResourceNotFoundException re) { - this.setResourceInformation(ResourceInformation.VCORES.getName(), - ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), - (long) vCores)); - } + setResourceInformation(ResourceInformation.VCORES.getName(), + ResourceInformation.newInstance(ResourceInformation.VCORES.getName(), + ResourceInformation.VCORES.getUnits(), (long) vCores)); } private void initResources() { @@ -132,14 +132,16 @@ public class ResourcePBImpl extends Resource { Long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation.newInstance(entry.getKey(), units, value, type); - resources.put(ri.getName(), ri); - } - if(this.getMemory() != p.getMemory()) { - setMemorySize(p.getMemory()); - } - if(this.getVirtualCores() != p.getVirtualCores()) { - setVirtualCores(p.getVirtualCores()); + if (resources.containsKey(ri.getName())) { + resources.get(ri.getName()).setResourceType(ri.getResourceType()); + resources.get(ri.getName()).setUnits(ri.getUnits()); + resources.get(ri.getName()).setValue(value); + } else { + LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping"); + } } + this.setMemorySize(p.getMemory()); + this.setVirtualCores(p.getVirtualCores()); } @Override @@ -153,7 +155,7 @@ public class ResourcePBImpl extends Resource { if (!resource.equals(resourceInformation.getName())) { resourceInformation.setName(resource); } - initResourcesMap(); + initResources(); resources.put(resource, resourceInformation); } @@ -161,6 +163,7 @@ public class ResourcePBImpl extends Resource { public void setResourceValue(String resource, Long value) throws ResourceNotFoundException { maybeInitBuilder(); + initResources(); if (resource == null) { throw new IllegalArgumentException("resource type object cannot be null"); } @@ -168,9 +171,7 @@ public class ResourcePBImpl extends Resource { throw new ResourceNotFoundException( "Resource " + resource + " not found"); } - ResourceInformation ri = resources.get(resource); - ri.setValue(value); - resources.put(resource, ri); + resources.get(resource).setValue(value); } @Override @@ -215,8 +216,10 @@ public class ResourcePBImpl extends Resource { synchronized private void mergeLocalToBuilder() { builder.clearResourceValueMap(); if (resources != null && !resources.isEmpty()) { - for (Map.Entry entry : resources.entrySet()) { - ResourceInformationProto.Builder e = ResourceInformationProto.newBuilder(); + for (Map.Entry entry : + resources.entrySet()) { + ResourceInformationProto.Builder e = + ResourceInformationProto.newBuilder(); e.setKey(entry.getKey()); e.setUnits(entry.getValue().getUnits()); e.setType( http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 0316f01..4f56fbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.util.resource; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -51,15 +52,21 @@ public class ResourceUtils { public static final String UNITS = ".units"; public static final String TYPE = ".type"; + private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); + private static final String VCORES = ResourceInformation.VCORES.getName(); + private static final Set DISALLOWED_NAMES = new HashSet<>(); static { DISALLOWED_NAMES.add("memory"); - DISALLOWED_NAMES.add(ResourceInformation.MEMORY_MB.getName()); - DISALLOWED_NAMES.add(ResourceInformation.VCORES.getName()); + DISALLOWED_NAMES.add(MEMORY); + DISALLOWED_NAMES.add(VCORES); } private static volatile Object lock; private static Map readOnlyResources; + private static volatile Object nodeLock; + private static Map readOnlyNodeResources; + static final Log LOG = LogFactory.getLog(ResourceUtils.class); @@ -69,22 +76,20 @@ public class ResourceUtils { private static void checkMandatatoryResources( Map resourceInformationMap) throws YarnRuntimeException { - String memory = ResourceInformation.MEMORY_MB.getName(); - String vcores = ResourceInformation.VCORES.getName(); - if (resourceInformationMap.containsKey(memory)) { - ResourceInformation memInfo = resourceInformationMap.get(memory); + if (resourceInformationMap.containsKey(MEMORY)) { + ResourceInformation memInfo = resourceInformationMap.get(MEMORY); String memUnits = ResourceInformation.MEMORY_MB.getUnits(); ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType(); if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType() .equals(memType)) { throw new YarnRuntimeException( "Attempt to re-define mandatory resource 'memory-mb'. It can only" - + " be of type 'COUNTABLE' and have units 'M'."); + + " be of type 'COUNTABLE' and have units 'Mi'."); } } - if (resourceInformationMap.containsKey(vcores)) { - ResourceInformation vcoreInfo = resourceInformationMap.get(vcores); + if (resourceInformationMap.containsKey(VCORES)) { + ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES); String vcoreUnits = ResourceInformation.VCORES.getUnits(); ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType(); if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo @@ -99,21 +104,21 @@ public class ResourceUtils { private static void addManadtoryResources( Map res) { ResourceInformation ri; - if (!res.containsKey(ResourceInformation.MEMORY_MB.getName())) { - LOG.info("Adding resource type - name = " + ResourceInformation.MEMORY_MB - .getName() + ", units = " + ResourceInformation.MEMORY_MB.getUnits() - + ", type = " + ResourceTypes.COUNTABLE); + if (!res.containsKey(MEMORY)) { + LOG.info("Adding resource type - name = " + MEMORY + ", units = " + + ResourceInformation.MEMORY_MB.getUnits() + ", type = " + + ResourceTypes.COUNTABLE); ri = ResourceInformation - .newInstance(ResourceInformation.MEMORY_MB.getName(), + .newInstance(MEMORY, ResourceInformation.MEMORY_MB.getUnits()); - res.put(ResourceInformation.MEMORY_MB.getName(), ri); + res.put(MEMORY, ri); } - if (!res.containsKey(ResourceInformation.VCORES.getName())) { - LOG.info("Adding resource type - name = " + ResourceInformation.VCORES - .getName() + ", units = , type = " + ResourceTypes.COUNTABLE); + if (!res.containsKey(VCORES)) { + LOG.info("Adding resource type - name = " + VCORES + ", units = , type = " + + ResourceTypes.COUNTABLE); ri = - ResourceInformation.newInstance(ResourceInformation.VCORES.getName()); - res.put(ResourceInformation.VCORES.getName(), ri); + ResourceInformation.newInstance(VCORES); + res.put(VCORES, ri); } } @@ -122,6 +127,7 @@ public class ResourceUtils { Map resourceInformationMap) { String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); + if (resourceNames != null && resourceNames.length != 0) { for (String resourceName : resourceNames) { String resourceUnits = conf.get( @@ -178,25 +184,13 @@ public class ResourceUtils { conf = new YarnConfiguration(); } try { - InputStream ris = getConfInputStream(resourceFile, conf); + addResourcesFileToConf(resourceFile, conf); LOG.debug("Found " + resourceFile + ", adding to configuration"); - conf.addResource(ris); initializeResourcesMap(conf, resources); - return resources; } catch (FileNotFoundException fe) { LOG.info("Unable to find '" + resourceFile + "'. Falling back to memory and vcores as resources", fe); initializeResourcesMap(conf, resources); - } catch (IOException ie) { - LOG.fatal( - "Exception trying to read resource types configuration '" - + resourceFile + "'.", ie); - throw new YarnRuntimeException(ie); - } catch (YarnException ye) { - LOG.fatal( - "YARN Exception trying to read resource types configuration '" - + resourceFile + "'.", ye); - throw new YarnRuntimeException(ye); } } } @@ -205,8 +199,8 @@ public class ResourceUtils { return readOnlyResources; } - static InputStream getConfInputStream(String resourceFile, Configuration conf) - throws IOException, YarnException { + private static InputStream getConfInputStream(String resourceFile, + Configuration conf) throws IOException, YarnException { ConfigurationProvider provider = ConfigurationProviderFactory.getConfigurationProvider(conf); @@ -222,8 +216,112 @@ public class ResourceUtils { return ris; } + private static void addResourcesFileToConf(String resourceFile, + Configuration conf) throws FileNotFoundException { + try { + InputStream ris = getConfInputStream(resourceFile, conf); + LOG.debug("Found " + resourceFile + ", adding to configuration"); + conf.addResource(ris); + } catch (FileNotFoundException fe) { + throw fe; + } catch (IOException ie) { + LOG.fatal("Exception trying to read resource types configuration '" + + resourceFile + "'.", ie); + throw new YarnRuntimeException(ie); + } catch (YarnException ye) { + LOG.fatal("YARN Exception trying to read resource types configuration '" + + resourceFile + "'.", ye); + throw new YarnRuntimeException(ye); + } + } + @VisibleForTesting static void resetResourceTypes() { lock = null; } + + private static String getUnits(String resourceValue) { + String units; + for (int i = 0; i < resourceValue.length(); i++) { + if (Character.isAlphabetic(resourceValue.charAt(i))) { + units = resourceValue.substring(i); + if (StringUtils.isAlpha(units)) { + return units; + } + } + } + return ""; + } + + /** + * Function to get the resources for a node. This function will look at the + * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to + * determine the node resources. + * + * @param conf configuration file + * @return a map to resource name to the ResourceInformation object. The map + * is guaranteed to have entries for memory and vcores + */ + public static Map getNodeResourceInformation( + Configuration conf) { + if (nodeLock == null) { + synchronized (ResourceUtils.class) { + if (nodeLock == null) { + synchronized (ResourceUtils.class) { + nodeLock = new Object(); + Map nodeResources = + initializeNodeResourceInformation(conf); + addManadtoryResources(nodeResources); + checkMandatatoryResources(nodeResources); + readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); + } + } + } + } + return readOnlyNodeResources; + } + + private static Map + initializeNodeResourceInformation(Configuration conf) { + Map nodeResources = new HashMap<>(); + try { + addResourcesFileToConf( + YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf); + for (Map.Entry entry : conf) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { + addResourceInformation(key, value, nodeResources); + } + } + } catch (FileNotFoundException fe) { + LOG.info("Couldn't find node resources file"); + } + return nodeResources; + } + + private static void addResourceInformation(String prop, String value, + Map nodeResources) { + String[] parts = prop.split("\\."); + LOG.info("Found resource entry " + prop); + if (parts.length == 4) { + String resourceType = parts[3]; + if (!nodeResources.containsKey(resourceType)) { + nodeResources + .put(resourceType, ResourceInformation.newInstance(resourceType)); + } + String units = getUnits(value); + Long resourceValue = + Long.valueOf(value.substring(0, value.length() - units.length())); + nodeResources.get(resourceType).setValue(resourceValue); + nodeResources.get(resourceType).setUnits(units); + LOG.debug("Setting value for resource type " + resourceType + " to " + + resourceValue + " with units " + units); + } + } + + @VisibleForTesting + synchronized public static void resetNodeResources() { + nodeLock = null; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 4bb68c0..ab014dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.util.SystemClock; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -245,4 +245,31 @@ public class TestResourceUtils { } } } + + @Test + public void testGetResourceInformation() throws Exception { + + Configuration conf = new YarnConfiguration(); + Map testRun = new HashMap<>(); + // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); + Resource test3Resources = Resource.newInstance(1024, 1); + test3Resources.setResourceInformation("resource1", + ResourceInformation.newInstance("resource1", "Gi", 5L)); + test3Resources.setResourceInformation("resource2", + ResourceInformation.newInstance("resource2", "m", 2L)); + testRun.put("node-resources-2.xml", test3Resources); + + for (Map.Entry entry : testRun.entrySet()) { + String resourceFile = entry.getKey(); + ResourceUtils.resetNodeResources(); + File dest; + File source = + new File(conf.getClassLoader().getResource(resourceFile).getFile()); + dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = + ResourceUtils.getNodeResourceInformation(conf); + Assert.assertEquals(entry.getValue().getResources(), actual); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml new file mode 100644 index 0000000..f00573e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-1.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.nodemanager.resource.memory-mb + 1024 + + + + yarn.nodemanager.resource.vcores + 1 + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml new file mode 100644 index 0000000..9d9b3dc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml @@ -0,0 +1,39 @@ + + + + + + + + yarn.nodemanager.resource-type.memory-mb + 1024Mi + + + + yarn.nodemanager.resource-type.vcores + 1 + + + + yarn.nodemanager.resource-type.resource1 + 5Gi + + + + yarn.nodemanager.resource-type.resource2 + 2m + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 7f74ed8..16c8cad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -173,18 +173,19 @@ public class NodeStatusUpdaterImpl extends AbstractService implements @Override protected void serviceInit(Configuration conf) throws Exception { - int memoryMb = NodeManagerHardwareUtils.getContainerMemoryMB(conf); + this.totalResource = NodeManagerHardwareUtils.getNodeResources(conf); + int memoryMb = totalResource.getMemory(); float vMemToPMem = conf.getFloat( YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); int virtualMemoryMb = (int)Math.ceil(memoryMb * vMemToPMem); - int virtualCores = NodeManagerHardwareUtils.getVCores(conf); + int virtualCores = totalResource.getVirtualCores(); LOG.info("Nodemanager resources: memory set to " + memoryMb + "MB."); LOG.info("Nodemanager resources: vcores set to " + virtualCores + "."); + LOG.info("Nodemanager resources: " + totalResource); - this.totalResource = Resource.newInstance(memoryMb, virtualCores); metrics.addResource(totalResource); // Get actual node physical resources http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java index 2726a41..3057770 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java @@ -23,8 +23,14 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; + +import java.util.Map; /** * Helper class to determine hardware related characteristics such as the @@ -332,4 +338,50 @@ public class NodeManagerHardwareUtils { } return memoryMb; } + + /** + * Get the resources for the node. + * @param configuration configuration file + * @return the resources for the node + */ + public static Resource getNodeResources(Configuration configuration) { + Configuration conf = new Configuration(configuration); + String memory = ResourceInformation.MEMORY_MB.getName(); + String vcores = ResourceInformation.VCORES.getName(); + + Resource ret = Resource.newInstance(0, 0); + Map resourceInformation = + ResourceUtils.getNodeResourceInformation(conf); + for (Map.Entry entry : resourceInformation + .entrySet()) { + ret.setResourceInformation(entry.getKey(), entry.getValue()); + LOG.debug("Setting key " + entry.getKey() + " to " + entry.getValue()); + } + if (resourceInformation.containsKey(memory)) { + Long value = resourceInformation.get(memory).getValue(); + if (value > Integer.MAX_VALUE) { + throw new YarnRuntimeException("Value '" + value + + "' for resource memory is more than the maximum for an integer."); + } + ResourceInformation memResInfo = resourceInformation.get(memory); + if(memResInfo.getValue() == 0) { + ret.setMemory(getContainerMemoryMB(conf)); + LOG.debug("Set memory to " + ret.getMemory()); + } + } + if (resourceInformation.containsKey(vcores)) { + Long value = resourceInformation.get(vcores).getValue(); + if (value > Integer.MAX_VALUE) { + throw new YarnRuntimeException("Value '" + value + + "' for resource vcores is more than the maximum for an integer."); + } + ResourceInformation vcoresResInfo = resourceInformation.get(vcores); + if(vcoresResInfo.getValue() == 0) { + ret.setVirtualCores(getVCores(conf)); + LOG.debug("Set vcores to " + ret.getVirtualCores()); + } + } + LOG.debug("Node resource information map is " + ret); + return ret; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d702592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 40bd610..1f291a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -372,10 +372,11 @@ public class ResourceTrackerService extends AbstractService implements // Check if this node has minimum allocations if (capability.getMemorySize() < minAllocMb || capability.getVirtualCores() < minAllocVcores) { - String message = - "NodeManager from " + host - + " doesn't satisfy minimum allocations, Sending SHUTDOWN" - + " signal to the NodeManager."; + String message = "NodeManager from " + host + + " doesn't satisfy minimum allocations, Sending SHUTDOWN" + + " signal to the NodeManager. Node capabilities are " + capability + + "; minimums are " + minAllocMb + "mb and " + minAllocVcores + + " vcores"; LOG.info(message); response.setDiagnosticsMessage(message); response.setNodeAction(NodeAction.SHUTDOWN); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org