Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 01A292004A0 for ; Wed, 16 Aug 2017 20:21:11 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id F415916931B; Wed, 16 Aug 2017 18:21:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 572751692FE for ; Wed, 16 Aug 2017 20:21:09 +0200 (CEST) Received: (qmail 65157 invoked by uid 500); 16 Aug 2017 18:20:53 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 61100 invoked by uid 99); 16 Aug 2017 18:20:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Aug 2017 18:20:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DDA1AF5EF4; Wed, 16 Aug 2017 18:20:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sunilg@apache.org To: common-commits@hadoop.apache.org Date: Wed, 16 Aug 2017 18:21:32 -0000 Message-Id: <782e0c058e434a7fb1c7f1db73158788@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [45/50] [abbrv] hadoop git commit: YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton) archived-at: Wed, 16 Aug 2017 18:21:11 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java deleted file mode 100644 index 86cf872..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ /dev/null @@ -1,488 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.util.resource; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; -import org.apache.hadoop.yarn.conf.ConfigurationProvider; -import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * Helper class to read the resource-types to be supported by the system. - */ -@InterfaceAudience.Public -@InterfaceStability.Unstable -public class ResourceUtils { - - public static final String UNITS = ".units"; - public static final String TYPE = ".type"; - public static final String MINIMUM_ALLOCATION = ".minimum-allocation"; - public static final String MAXIMUM_ALLOCATION = ".maximum-allocation"; - - private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); - private static final String VCORES = ResourceInformation.VCORES.getName(); - - private static final Set DISALLOWED_NAMES = new HashSet<>(); - static { - DISALLOWED_NAMES.add("memory"); - DISALLOWED_NAMES.add(MEMORY); - DISALLOWED_NAMES.add(VCORES); - } - - private static volatile Object lock; - private static Map readOnlyResources; - private static volatile Object nodeLock; - private static Map readOnlyNodeResources; - - - static final Log LOG = LogFactory.getLog(ResourceUtils.class); - - private ResourceUtils() { - } - - private static void checkMandatatoryResources( - Map resourceInformationMap) - throws YarnRuntimeException { - if (resourceInformationMap.containsKey(MEMORY)) { - ResourceInformation memInfo = resourceInformationMap.get(MEMORY); - String memUnits = ResourceInformation.MEMORY_MB.getUnits(); - ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType(); - if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType() - .equals(memType)) { - throw new YarnRuntimeException( - "Attempt to re-define mandatory resource 'memory-mb'. It can only" - + " be of type 'COUNTABLE' and have units 'Mi'."); - } - } - - if (resourceInformationMap.containsKey(VCORES)) { - ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES); - String vcoreUnits = ResourceInformation.VCORES.getUnits(); - ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType(); - if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo - .getResourceType().equals(vcoreType)) { - throw new YarnRuntimeException( - "Attempt to re-define mandatory resource 'vcores'. It can only be" - + " of type 'COUNTABLE' and have units ''(no units)."); - } - } - } - - private static void addManadtoryResources( - Map res) { - ResourceInformation ri; - if (!res.containsKey(MEMORY)) { - LOG.info("Adding resource type - name = " + MEMORY + ", units = " - + ResourceInformation.MEMORY_MB.getUnits() + ", type = " - + ResourceTypes.COUNTABLE); - ri = ResourceInformation - .newInstance(MEMORY, - ResourceInformation.MEMORY_MB.getUnits()); - res.put(MEMORY, ri); - } - if (!res.containsKey(VCORES)) { - LOG.info("Adding resource type - name = " + VCORES + ", units = , type = " - + ResourceTypes.COUNTABLE); - ri = - ResourceInformation.newInstance(VCORES); - res.put(VCORES, ri); - } - } - - private static void setMinimumAllocationForMandatoryResources( - Map res, Configuration conf) { - String[][] resourceTypesKeys = - { - { ResourceInformation.MEMORY_MB.getName(), - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB), - ResourceInformation.MEMORY_MB.getName() - }, - { ResourceInformation.VCORES.getName(), - YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES), - ResourceInformation.VCORES.getName() - } - }; - for (String[] arr : resourceTypesKeys) { - String resourceTypesKey = - YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION; - long minimumResourceTypes = conf.getLong(resourceTypesKey, -1); - long minimumConf = conf.getLong(arr[1], -1); - long minimum; - if (minimumResourceTypes != -1) { - minimum = minimumResourceTypes; - if (minimumConf != -1) { - LOG.warn("Using minimum allocation for memory specified in " - + "resource-types config file with key " - + minimumResourceTypes + ", ignoring minimum specified using " - + arr[1]); - } - } else { - minimum = conf.getLong(arr[1], Long.parseLong(arr[2])); - } - ResourceInformation ri = res.get(arr[3]); - ri.setMinimumAllocation(minimum); - } - } - - private static void setMaximumAllocationForMandatoryResources( - Map res, Configuration conf) { - String[][] resourceTypesKeys = - { - { - ResourceInformation.MEMORY_MB.getName(), - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), - ResourceInformation.MEMORY_MB.getName() - }, - { - ResourceInformation.VCORES.getName(), - YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, - String.valueOf( - YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES), - ResourceInformation.VCORES.getName() - } - }; - for (String[] arr : resourceTypesKeys) { - String resourceTypesKey = - YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION; - long maximumResourceTypes = conf.getLong(resourceTypesKey, -1); - long maximumConf = conf.getLong(arr[1], -1); - long maximum; - if (maximumResourceTypes != -1) { - maximum = maximumResourceTypes; - if (maximumConf != -1) { - LOG.warn("Using maximum allocation for memory specified in " - + "resource-types config file with key " - + maximumResourceTypes + ", ignoring maximum specified using " - + arr[1]); - } - } else { - maximum = conf.getLong(arr[1], Long.parseLong(arr[2])); - } - ResourceInformation ri = res.get(arr[3]); - ri.setMaximumAllocation(maximum); - } - } - - @VisibleForTesting - static void initializeResourcesMap(Configuration conf, - Map resourceInformationMap) { - - String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES); - - if (resourceNames != null && resourceNames.length != 0) { - for (String resourceName : resourceNames) { - String resourceUnits = conf.get( - YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, ""); - String resourceTypeName = conf.get( - YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE, - ResourceTypes.COUNTABLE.toString()); - Long minimumAllocation = conf.getLong( - YarnConfiguration.RESOURCE_TYPES + "." + resourceName - + MINIMUM_ALLOCATION, 0L); - Long maximumAllocation = conf.getLong( - YarnConfiguration.RESOURCE_TYPES + "." + resourceName - + MAXIMUM_ALLOCATION, Long.MAX_VALUE); - if (resourceName == null || resourceName.isEmpty() - || resourceUnits == null || resourceTypeName == null) { - throw new YarnRuntimeException( - "Incomplete configuration for resource type '" + resourceName - + "'. One of name, units or type is configured incorrectly."); - } - if (DISALLOWED_NAMES.contains(resourceName)) { - throw new YarnRuntimeException( - "Resource type cannot be named '" + resourceName - + "'. That name is disallowed."); - } - ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName); - LOG.info("Adding resource type - name = " + resourceName + ", units = " - + resourceUnits + ", type = " + resourceTypeName); - if (resourceInformationMap.containsKey(resourceName)) { - throw new YarnRuntimeException( - "Error in config, key '" + resourceName + "' specified twice"); - } - resourceInformationMap.put(resourceName, ResourceInformation - .newInstance(resourceName, resourceUnits, 0L, resourceType, - minimumAllocation, maximumAllocation)); - } - } - checkMandatatoryResources(resourceInformationMap); - addManadtoryResources(resourceInformationMap); - setMinimumAllocationForMandatoryResources(resourceInformationMap, conf); - setMaximumAllocationForMandatoryResources(resourceInformationMap, conf); - readOnlyResources = Collections.unmodifiableMap(resourceInformationMap); - } - - /** - * Get the resource types to be supported by the system. - * @return A map of the resource name to a ResouceInformation object - * which contains details such as the unit. - */ - public static Map getResourceTypes() { - return getResourceTypes(null, - YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); - } - - private static Map getResourceTypes( - Configuration conf) { - return getResourceTypes(conf, - YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE); - } - - private static Map getResourceTypes( - Configuration conf, String resourceFile) { - if (lock == null) { - synchronized (ResourceUtils.class) { - if (lock == null) { - synchronized (ResourceUtils.class) { - Map resources = new HashMap<>(); - if (conf == null) { - conf = new YarnConfiguration(); - } - try { - addResourcesFileToConf(resourceFile, conf); - LOG.debug("Found " + resourceFile + ", adding to configuration"); - initializeResourcesMap(conf, resources); - lock = new Object(); - } catch (FileNotFoundException fe) { - LOG.info("Unable to find '" + resourceFile - + "'. Falling back to memory and vcores as resources", fe); - initializeResourcesMap(conf, resources); - lock = new Object(); - } - } - } - } - } - return readOnlyResources; - } - - private static InputStream getConfInputStream(String resourceFile, - Configuration conf) throws IOException, YarnException { - - ConfigurationProvider provider = - ConfigurationProviderFactory.getConfigurationProvider(conf); - try { - provider.init(conf); - } catch (Exception e) { - throw new IOException(e); - } - - InputStream ris = provider.getConfigurationInputStream(conf, resourceFile); - if (ris == null) { - if (conf.getResource(resourceFile) == null) { - throw new FileNotFoundException("Unable to find " + resourceFile); - } - throw new IOException( - "Unable to open resource types file '" + resourceFile - + "'. Using provider " + provider); - } - return ris; - } - - private static void addResourcesFileToConf(String resourceFile, - Configuration conf) throws FileNotFoundException { - try { - InputStream ris = getConfInputStream(resourceFile, conf); - LOG.debug("Found " + resourceFile + ", adding to configuration"); - conf.addResource(ris); - } catch (FileNotFoundException fe) { - throw fe; - } catch (IOException ie) { - LOG.fatal("Exception trying to read resource types configuration '" - + resourceFile + "'.", ie); - throw new YarnRuntimeException(ie); - } catch (YarnException ye) { - LOG.fatal("YARN Exception trying to read resource types configuration '" - + resourceFile + "'.", ye); - throw new YarnRuntimeException(ye); - } - } - - @VisibleForTesting - static void resetResourceTypes() { - lock = null; - } - - @VisibleForTesting - public static void resetResourceTypes(Configuration conf) { - lock = null; - getResourceTypes(conf); - } - - public static String getUnits(String resourceValue) { - String units; - for (int i = 0; i < resourceValue.length(); i++) { - if (Character.isAlphabetic(resourceValue.charAt(i))) { - units = resourceValue.substring(i); - if (StringUtils.isAlpha(units)) { - return units; - } - } - } - return ""; - } - - /** - * Function to get the resources for a node. This function will look at the - * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to - * determine the node resources. - * - * @param conf configuration file - * @return a map to resource name to the ResourceInformation object. The map - * is guaranteed to have entries for memory and vcores - */ - public static Map getNodeResourceInformation( - Configuration conf) { - if (nodeLock == null) { - synchronized (ResourceUtils.class) { - if (nodeLock == null) { - synchronized (ResourceUtils.class) { - Map nodeResources = - initializeNodeResourceInformation(conf); - addManadtoryResources(nodeResources); - checkMandatatoryResources(nodeResources); - readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); - nodeLock = new Object(); - } - } - } - } - return readOnlyNodeResources; - } - - private static Map - initializeNodeResourceInformation(Configuration conf) { - Map nodeResources = new HashMap<>(); - try { - addResourcesFileToConf( - YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf); - for (Map.Entry entry : conf) { - String key = entry.getKey(); - String value = entry.getValue(); - if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) { - addResourceInformation(key, value, nodeResources); - } - } - } catch (FileNotFoundException fe) { - LOG.info("Couldn't find node resources file"); - } - return nodeResources; - } - - private static void addResourceInformation(String prop, String value, - Map nodeResources) { - String[] parts = prop.split("\\."); - LOG.info("Found resource entry " + prop); - if (parts.length == 4) { - String resourceType = parts[3]; - if (!nodeResources.containsKey(resourceType)) { - nodeResources - .put(resourceType, ResourceInformation.newInstance(resourceType)); - } - String units = getUnits(value); - Long resourceValue = - Long.valueOf(value.substring(0, value.length() - units.length())); - nodeResources.get(resourceType).setValue(resourceValue); - nodeResources.get(resourceType).setUnits(units); - LOG.debug("Setting value for resource type " + resourceType + " to " - + resourceValue + " with units " + units); - } - } - - @VisibleForTesting - synchronized public static void resetNodeResources() { - nodeLock = null; - } - - public static Resource getResourceTypesMinimumAllocation() { - Map resourceTypes = getResourceTypes(); - Resource ret = Resource.newInstance(0, 0); - for (Map.Entry entry : resourceTypes - .entrySet()) { - String name = entry.getKey(); - if (name.equals(ResourceInformation.MEMORY_MB.getName())) { - ret.setMemorySize(entry.getValue().getMinimumAllocation()); - continue; - } - if (name.equals(ResourceInformation.VCORES.getName())) { - Long tmp = entry.getValue().getMinimumAllocation(); - if (tmp > Integer.MAX_VALUE) { - tmp = (long) Integer.MAX_VALUE; - } - ret.setVirtualCores(tmp.intValue()); - continue; - } - ret.setResourceValue(name, entry.getValue().getMinimumAllocation()); - } - return ret; - } - - /** - * Get a Resource object with for the maximum allocation possible. - * @return a Resource object with the maximum allocation for the scheduler - */ - public static Resource getResourceTypesMaximumAllocation() { - Map resourceTypes = getResourceTypes(); - Resource ret = Resource.newInstance(0, 0); - for (Map.Entry entry : resourceTypes - .entrySet()) { - String name = entry.getKey(); - if (name.equals(ResourceInformation.MEMORY_MB.getName())) { - ret.setMemorySize(entry.getValue().getMaximumAllocation()); - continue; - } - if (name.equals(ResourceInformation.VCORES.getName())) { - Long tmp = entry.getValue().getMaximumAllocation(); - if (tmp > Integer.MAX_VALUE) { - tmp = (long) Integer.MAX_VALUE; - } - ret.setVirtualCores(tmp.intValue()); - continue; - } - ret.setResourceValue(name, entry.getValue().getMaximumAllocation()); - } - return ret; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index d143e93..f62114d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -18,30 +18,31 @@ package org.apache.hadoop.yarn.util.resource; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.impl.BaseResource; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.UnitsConversionUtil; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.Arrays; @InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" }) @Unstable public class Resources { + private static final Log LOG = + LogFactory.getLog(Resources.class); + /** * Helper class to create a resource with a fixed value for all resource * types. For example, a NONE resource which returns 0 for any resource type. */ - static class FixedValueResource extends Resource { + static class FixedValueResource extends BaseResource { - private Map resources; private long resourceValue; private String name; @@ -53,7 +54,7 @@ public class Resources { FixedValueResource(String rName, long value) { this.resourceValue = value; this.name = rName; - resources = initResourceMap(); + initResourceMap(); } private int resourceValueToInt() { @@ -96,31 +97,6 @@ public class Resources { } @Override - public Map getResources() { - return Collections.unmodifiableMap(this.resources); - } - - @Override - public ResourceInformation getResourceInformation(String resource) - throws YarnException { - if (resources.containsKey(resource)) { - ResourceInformation value = this.resources.get(resource); - ResourceInformation ret = ResourceInformation.newInstance(value); - ret.setValue(resourceValue); - return ret; - } - throw new YarnException("" + resource + " not found"); - } - - @Override - public Long getResourceValue(String resource) throws YarnException { - if (resources.containsKey(resource)) { - return resourceValue; - } - throw new YarnException("" + resource + " not found"); - } - - @Override public void setResourceInformation(String resource, ResourceInformation resourceInformation) throws ResourceNotFoundException { @@ -133,24 +109,24 @@ public class Resources { throw new RuntimeException(name + " cannot be modified!"); } - private Map initResourceMap() { - Map tmp = new HashMap<>(); - Map types = ResourceUtils.getResourceTypes(); + private void initResourceMap() { + ResourceInformation[] types = ResourceUtils.getResourceTypesArray(); if (types != null) { - for (Map.Entry entry : types.entrySet()) { - tmp.put(entry.getKey(), - ResourceInformation.newInstance(entry.getValue())); - tmp.get(entry.getKey()).setValue(resourceValue); + resources = new ResourceInformation[types.length]; + readOnlyResources = new ResourceInformation[types.length]; + for (int index = 0; index < types.length; index++) { + resources[index] = ResourceInformation.newInstance(types[index]); + resources[index].setValue(resourceValue); + + // this is a fix for getVirtualCores returning an int + if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES + .getName().equals(resources[index].getName())) { + resources[index].setValue((long) Integer.MAX_VALUE); + } } } - // this is a fix for getVirtualCores returning an int - if (resourceValue > Integer.MAX_VALUE) { - tmp.get(ResourceInformation.VCORES.getName()) - .setValue((long) Integer.MAX_VALUE); - } - return tmp; + readOnlyResources = Arrays.copyOf(resources, resources.length); } - } public static Resource createResource(int memory) { @@ -197,17 +173,19 @@ public class Resources { } public static Resource addTo(Resource lhs, Resource rhs) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); - long convertedRhs = UnitsConversionUtil - .convert(rhsValue.getUnits(), lhsValue.getUnits(), - rhsValue.getValue()); + ResourceInformation lhsValue = entry; + + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) + ? rhsValue.getValue() + : UnitsConversionUtil.convert(rhsValue.getUnits(), + lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } @@ -219,17 +197,19 @@ public class Resources { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); - long convertedRhs = UnitsConversionUtil - .convert(rhsValue.getUnits(), lhsValue.getUnits(), - rhsValue.getValue()); + ResourceInformation lhsValue = entry; + + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) + ? rhsValue.getValue() + : UnitsConversionUtil.convert(rhsValue.getUnits(), + lhsValue.getUnits(), rhsValue.getValue()); lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } @@ -263,10 +243,9 @@ public class Resources { } public static Resource multiplyTo(Resource lhs, double by) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); - ResourceInformation lhsValue = entry.getValue(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); + ResourceInformation lhsValue = entry; lhs.setResourceValue(name, (long) (lhsValue.getValue() * by)); } return lhs; @@ -282,17 +261,21 @@ public class Resources { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); - long convertedRhs = (long) (UnitsConversionUtil - .convert(rhsValue.getUnits(), lhsValue.getUnits(), - rhsValue.getValue()) * by); + ResourceInformation lhsValue = entry; + + long convertedRhs = (long) (((rhsValue.getUnits() + .equals(lhsValue.getUnits())) + ? rhsValue.getValue() + : UnitsConversionUtil.convert(rhsValue.getUnits(), + lhsValue.getUnits(), rhsValue.getValue())) + * by); lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } @@ -311,10 +294,9 @@ public class Resources { public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - for (Map.Entry entry : out.getResources() - .entrySet()) { - String name = entry.getKey(); - ResourceInformation lhsValue = entry.getValue(); + for (ResourceInformation entry : out.getResources()) { + String name = entry.getName(); + ResourceInformation lhsValue = entry; out.setResourceValue(name, (long) (lhsValue.getValue() * by)); } return out; @@ -416,19 +398,21 @@ public class Resources { } public static boolean fitsIn(Resource smaller, Resource bigger) { - for (Map.Entry entry : smaller.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : smaller.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = bigger.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); - long convertedRhs = UnitsConversionUtil - .convert(rhsValue.getUnits(), lhsValue.getUnits(), - rhsValue.getValue()); + ResourceInformation lhsValue = entry; + + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) + ? rhsValue.getValue() + : UnitsConversionUtil.convert(rhsValue.getUnits(), + lhsValue.getUnits(), rhsValue.getValue()); if(lhsValue.getValue() > convertedRhs) { return false; } - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); return false; } } @@ -442,19 +426,21 @@ public class Resources { public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); - long convertedRhs = UnitsConversionUtil - .convert(rhsValue.getUnits(), lhsValue.getUnits(), - rhsValue.getValue()); + ResourceInformation lhsValue = entry; + + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) + ? rhsValue.getValue() + : UnitsConversionUtil.convert(rhsValue.getUnits(), + lhsValue.getUnits(), rhsValue.getValue()); ResourceInformation outInfo = lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue; ret.setResourceInformation(name, outInfo); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } @@ -463,19 +449,21 @@ public class Resources { public static Resource componentwiseMax(Resource lhs, Resource rhs) { Resource ret = createResource(0); - for (Map.Entry entry : lhs.getResources() - .entrySet()) { - String name = entry.getKey(); + for (ResourceInformation entry : lhs.getResources()) { + String name = entry.getName(); try { ResourceInformation rhsValue = rhs.getResourceInformation(name); - ResourceInformation lhsValue = entry.getValue(); - long convertedRhs = UnitsConversionUtil - .convert(rhsValue.getUnits(), lhsValue.getUnits(), - rhsValue.getValue()); + ResourceInformation lhsValue = entry; + + long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits())) + ? rhsValue.getValue() + : UnitsConversionUtil.convert(rhsValue.getUnits(), + lhsValue.getUnits(), rhsValue.getValue()); ResourceInformation outInfo = lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue; ret.setResourceInformation(name, outInfo); - } catch (YarnException ye) { + } catch (ResourceNotFoundException ye) { + LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 38554b6..b530150 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -276,13 +276,17 @@ public class TestResourceUtils { String resourceFile = entry.getKey(); ResourceUtils.resetNodeResources(); File dest; - File source = - new File(conf.getClassLoader().getResource(resourceFile).getFile()); + File source = new File( + conf.getClassLoader().getResource(resourceFile).getFile()); dest = new File(source.getParent(), "node-resources.xml"); FileUtils.copyFile(source, dest); - Map actual = - ResourceUtils.getNodeResourceInformation(conf); - Assert.assertEquals(entry.getValue().getResources(), actual); + Map actual = ResourceUtils + .getNodeResourceInformation(conf); + Assert.assertEquals(actual.size(), + entry.getValue().getResources().length); + for (ResourceInformation resInfo : entry.getValue().getResources()) { + Assert.assertEquals(resInfo, actual.get(resInfo.getName())); + } dest.delete(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 1555e55..a8404fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -105,7 +105,7 @@ public class TestResources { unboundedClone.compareTo(createResource(0, Integer.MAX_VALUE)) > 0); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testCompareToWithNoneResource() { assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0); assertTrue(Resources.none().compareTo(createResource(1, 0)) < 0); @@ -114,7 +114,6 @@ public class TestResources { assertTrue(Resources.none().compareTo(createResource(1, 0, 0)) < 0); assertTrue(Resources.none().compareTo(createResource(0, 1, 0)) < 0); assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0); - assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0); } @Test(timeout=10000) @@ -246,7 +245,9 @@ public class TestResources { } @Test - public void testMultiplyAndAddTo() { + public void testMultiplyAndAddTo() throws Exception { + unsetExtraResourceType(); + setupExtraResourceType(); assertEquals(createResource(6, 4), multiplyAndAddTo(createResource(3, 1), createResource(2, 2), 1.5)); assertEquals(createResource(6, 4, 0), http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index 7987ded..ab33336 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -119,13 +119,13 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { private Resource parseResource(String key, Map value) throws IOException { Resource resource = Resource.newInstance(0, 0); Iterator iterator = value.entrySet().iterator(); - Map resourceTypes = - ResourceUtils.getResourceTypes(); + Map resourceTypes = ResourceUtils + .getResourceTypes(); while (iterator.hasNext()) { Map.Entry resourceEntry = (Map.Entry) iterator.next(); String resourceName = resourceEntry.getKey().toString(); - ResourceInformation resourceValue = - fromString(resourceName, resourceEntry.getValue().toString()); + ResourceInformation resourceValue = fromString(resourceName, + resourceEntry.getValue().toString()); if (resourceName.equals(MEMORY)) { resource.setMemorySize(resourceValue.getValue()); continue; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index ff18223..c514cb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -176,16 +176,15 @@ public class RMAppAttemptMetrics { private void updateUsageMap(Resource allocated, long deltaUsedMillis, Map targetMap) { - for (Map.Entry entry : allocated.getResources() - .entrySet()) { + for (ResourceInformation entry : allocated.getResources()) { AtomicLong resourceUsed; - if (!targetMap.containsKey(entry.getKey())) { + if (!targetMap.containsKey(entry.getName())) { resourceUsed = new AtomicLong(0); - targetMap.put(entry.getKey(), resourceUsed); + targetMap.put(entry.getName(), resourceUsed); } - resourceUsed = targetMap.get(entry.getKey()); - resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis) + resourceUsed = targetMap.get(entry.getName()); + resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis) / DateUtils.MILLIS_PER_SECOND); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index 92447e7..cf2e70d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -984,13 +984,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { for (RMContainer rmContainer : this.liveContainers.values()) { long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); Resource resource = rmContainer.getContainer().getResource(); - for (Map.Entry entry : resource - .getResources().entrySet()) { + for (ResourceInformation entry : resource.getResources()) { long value = RMServerUtils - .getOrDefault(resourceSecondsMap, entry.getKey(), 0L); - value += entry.getValue().getValue() * usedMillis + .getOrDefault(resourceSecondsMap, entry.getName(), 0L); + value += entry.getValue() * usedMillis / DateUtils.MILLIS_PER_SECOND; - resourceSecondsMap.put(entry.getKey(), value); + resourceSecondsMap.put(entry.getName(), value); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java index 887b854..81491b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; +import java.util.Arrays; import java.util.EnumSet; import javax.xml.bind.annotation.XmlRootElement; @@ -73,7 +74,7 @@ public class SchedulerInfo { } public String getSchedulerResourceTypes() { - return minAllocResource.getResource().getResources().keySet().toString(); + return Arrays.toString(minAllocResource.getResource().getResources()); } public int getMaxClusterLevelAppPriority() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f84812c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 8ae630b..b24a309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -249,6 +249,7 @@ public class TestAppManager{ ResourceScheduler scheduler = mockResourceScheduler(); ((RMContextImpl)rmContext).setScheduler(scheduler); Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); ((RMContextImpl) rmContext).setYarnConfiguration(conf); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org