hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From templ...@apache.org
Subject [33/50] [abbrv] hadoop git commit: YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton)
Date Mon, 23 Oct 2017 18:20:34 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
deleted file mode 100644
index 86cf872..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util.resource;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.ConfigurationProvider;
-import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Helper class to read the resource-types to be supported by the system.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ResourceUtils {
-
-  public static final String UNITS = ".units";
-  public static final String TYPE = ".type";
-  public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
-  public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
-
-  private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
-  private static final String VCORES = ResourceInformation.VCORES.getName();
-
-  private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
-  static {
-    DISALLOWED_NAMES.add("memory");
-    DISALLOWED_NAMES.add(MEMORY);
-    DISALLOWED_NAMES.add(VCORES);
-  }
-
-  private static volatile Object lock;
-  private static Map<String, ResourceInformation> readOnlyResources;
-  private static volatile Object nodeLock;
-  private static Map<String, ResourceInformation> readOnlyNodeResources;
-
-
-  static final Log LOG = LogFactory.getLog(ResourceUtils.class);
-
-  private ResourceUtils() {
-  }
-
-  private static void checkMandatatoryResources(
-      Map<String, ResourceInformation> resourceInformationMap)
-      throws YarnRuntimeException {
-    if (resourceInformationMap.containsKey(MEMORY)) {
-      ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
-      String memUnits = ResourceInformation.MEMORY_MB.getUnits();
-      ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
-      if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
-          .equals(memType)) {
-        throw new YarnRuntimeException(
-            "Attempt to re-define mandatory resource 'memory-mb'. It can only"
-                + " be of type 'COUNTABLE' and have units 'Mi'.");
-      }
-    }
-
-    if (resourceInformationMap.containsKey(VCORES)) {
-      ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
-      String vcoreUnits = ResourceInformation.VCORES.getUnits();
-      ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
-      if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
-          .getResourceType().equals(vcoreType)) {
-        throw new YarnRuntimeException(
-            "Attempt to re-define mandatory resource 'vcores'. It can only be"
-                + " of type 'COUNTABLE' and have units ''(no units).");
-      }
-    }
-  }
-
-  private static void addManadtoryResources(
-      Map<String, ResourceInformation> res) {
-    ResourceInformation ri;
-    if (!res.containsKey(MEMORY)) {
-      LOG.info("Adding resource type - name = " + MEMORY + ", units = "
-          + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
-          + ResourceTypes.COUNTABLE);
-      ri = ResourceInformation
-          .newInstance(MEMORY,
-              ResourceInformation.MEMORY_MB.getUnits());
-      res.put(MEMORY, ri);
-    }
-    if (!res.containsKey(VCORES)) {
-      LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
-          + ResourceTypes.COUNTABLE);
-      ri =
-          ResourceInformation.newInstance(VCORES);
-      res.put(VCORES, ri);
-    }
-  }
-
-  private static void setMinimumAllocationForMandatoryResources(
-      Map<String, ResourceInformation> res, Configuration conf) {
-    String[][] resourceTypesKeys =
-        {
-          { ResourceInformation.MEMORY_MB.getName(),
-            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
-            String.valueOf(
-              YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
-            ResourceInformation.MEMORY_MB.getName()
-          },
-          { ResourceInformation.VCORES.getName(),
-            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
-            String.valueOf(
-              YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
-            ResourceInformation.VCORES.getName()
-          }
-        };
-    for (String[] arr : resourceTypesKeys) {
-      String resourceTypesKey =
-          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
-      long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
-      long minimumConf = conf.getLong(arr[1], -1);
-      long minimum;
-      if (minimumResourceTypes != -1) {
-        minimum = minimumResourceTypes;
-        if (minimumConf != -1) {
-          LOG.warn("Using minimum allocation for memory specified in "
-              + "resource-types config file with key "
-              + minimumResourceTypes + ", ignoring minimum specified using "
-              + arr[1]);
-        }
-      } else {
-        minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
-      }
-      ResourceInformation ri = res.get(arr[3]);
-      ri.setMinimumAllocation(minimum);
-    }
-  }
-
-  private static void setMaximumAllocationForMandatoryResources(
-      Map<String, ResourceInformation> res, Configuration conf) {
-    String[][] resourceTypesKeys =
-        {
-          {
-            ResourceInformation.MEMORY_MB.getName(),
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            String.valueOf(
-              YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
-            ResourceInformation.MEMORY_MB.getName()
-          },
-          {
-            ResourceInformation.VCORES.getName(),
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            String.valueOf(
-              YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
-             ResourceInformation.VCORES.getName()
-          }
-        };
-    for (String[] arr : resourceTypesKeys) {
-      String resourceTypesKey =
-          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
-      long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
-      long maximumConf = conf.getLong(arr[1], -1);
-      long maximum;
-      if (maximumResourceTypes != -1) {
-        maximum = maximumResourceTypes;
-        if (maximumConf != -1) {
-          LOG.warn("Using maximum allocation for memory specified in "
-              + "resource-types config file with key "
-              + maximumResourceTypes + ", ignoring maximum specified using "
-              + arr[1]);
-        }
-      } else {
-        maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
-      }
-      ResourceInformation ri = res.get(arr[3]);
-      ri.setMaximumAllocation(maximum);
-    }
-  }
-
-  @VisibleForTesting
-  static void initializeResourcesMap(Configuration conf,
-      Map<String, ResourceInformation> resourceInformationMap) {
-
-    String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
-
-    if (resourceNames != null && resourceNames.length != 0) {
-      for (String resourceName : resourceNames) {
-        String resourceUnits = conf.get(
-            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
-        String resourceTypeName = conf.get(
-            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
-            ResourceTypes.COUNTABLE.toString());
-        Long minimumAllocation = conf.getLong(
-            YarnConfiguration.RESOURCE_TYPES + "." + resourceName
-                + MINIMUM_ALLOCATION, 0L);
-        Long maximumAllocation = conf.getLong(
-            YarnConfiguration.RESOURCE_TYPES + "." + resourceName
-                + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
-        if (resourceName == null || resourceName.isEmpty()
-            || resourceUnits == null || resourceTypeName == null) {
-          throw new YarnRuntimeException(
-              "Incomplete configuration for resource type '" + resourceName
-                  + "'. One of name, units or type is configured incorrectly.");
-        }
-        if (DISALLOWED_NAMES.contains(resourceName)) {
-          throw new YarnRuntimeException(
-              "Resource type cannot be named '" + resourceName
-                  + "'. That name is disallowed.");
-        }
-        ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
-        LOG.info("Adding resource type - name = " + resourceName + ", units = "
-            + resourceUnits + ", type = " + resourceTypeName);
-        if (resourceInformationMap.containsKey(resourceName)) {
-          throw new YarnRuntimeException(
-              "Error in config, key '" + resourceName + "' specified twice");
-        }
-        resourceInformationMap.put(resourceName, ResourceInformation
-            .newInstance(resourceName, resourceUnits, 0L, resourceType,
-                minimumAllocation, maximumAllocation));
-      }
-    }
-    checkMandatatoryResources(resourceInformationMap);
-    addManadtoryResources(resourceInformationMap);
-    setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
-    setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
-    readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
-  }
-
-  /**
-   * Get the resource types to be supported by the system.
-   * @return A map of the resource name to a ResouceInformation object
-   *         which contains details such as the unit.
-   */
-  public static Map<String, ResourceInformation> getResourceTypes() {
-    return getResourceTypes(null,
-        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
-  }
-
-  private static Map<String, ResourceInformation> getResourceTypes(
-      Configuration conf) {
-    return getResourceTypes(conf,
-        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
-  }
-
-  private static Map<String, ResourceInformation> getResourceTypes(
-      Configuration conf, String resourceFile) {
-    if (lock == null) {
-      synchronized (ResourceUtils.class) {
-        if (lock == null) {
-          synchronized (ResourceUtils.class) {
-            Map<String, ResourceInformation> resources = new HashMap<>();
-            if (conf == null) {
-              conf = new YarnConfiguration();
-            }
-            try {
-              addResourcesFileToConf(resourceFile, conf);
-              LOG.debug("Found " + resourceFile + ", adding to configuration");
-              initializeResourcesMap(conf, resources);
-              lock = new Object();
-            } catch (FileNotFoundException fe) {
-              LOG.info("Unable to find '" + resourceFile
-                  + "'. Falling back to memory and vcores as resources", fe);
-              initializeResourcesMap(conf, resources);
-              lock = new Object();
-            }
-          }
-        }
-      }
-    }
-    return readOnlyResources;
-  }
-
-  private static InputStream getConfInputStream(String resourceFile,
-      Configuration conf) throws IOException, YarnException {
-
-    ConfigurationProvider provider =
-        ConfigurationProviderFactory.getConfigurationProvider(conf);
-    try {
-      provider.init(conf);
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
-
-    InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
-    if (ris == null) {
-      if (conf.getResource(resourceFile) == null) {
-        throw new FileNotFoundException("Unable to find " + resourceFile);
-      }
-      throw new IOException(
-          "Unable to open resource types file '" + resourceFile
-              + "'. Using provider " + provider);
-    }
-    return ris;
-  }
-
-  private static void addResourcesFileToConf(String resourceFile,
-      Configuration conf) throws FileNotFoundException {
-    try {
-      InputStream ris = getConfInputStream(resourceFile, conf);
-      LOG.debug("Found " + resourceFile + ", adding to configuration");
-      conf.addResource(ris);
-    } catch (FileNotFoundException fe) {
-      throw fe;
-    } catch (IOException ie) {
-      LOG.fatal("Exception trying to read resource types configuration '"
-          + resourceFile + "'.", ie);
-      throw new YarnRuntimeException(ie);
-    } catch (YarnException ye) {
-      LOG.fatal("YARN Exception trying to read resource types configuration '"
-          + resourceFile + "'.", ye);
-      throw new YarnRuntimeException(ye);
-    }
-  }
-
-  @VisibleForTesting
-  static void resetResourceTypes() {
-    lock = null;
-  }
-
-  @VisibleForTesting
-  public static void resetResourceTypes(Configuration conf) {
-    lock = null;
-    getResourceTypes(conf);
-  }
-
-  public static String getUnits(String resourceValue) {
-    String units;
-    for (int i = 0; i < resourceValue.length(); i++) {
-      if (Character.isAlphabetic(resourceValue.charAt(i))) {
-        units = resourceValue.substring(i);
-        if (StringUtils.isAlpha(units)) {
-          return units;
-        }
-      }
-    }
-    return "";
-  }
-
-  /**
-   * Function to get the resources for a node. This function will look at the
-   * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to
-   * determine the node resources.
-   *
-   * @param conf configuration file
-   * @return a map to resource name to the ResourceInformation object. The map
-   * is guaranteed to have entries for memory and vcores
-   */
-  public static Map<String, ResourceInformation> getNodeResourceInformation(
-      Configuration conf) {
-    if (nodeLock == null) {
-      synchronized (ResourceUtils.class) {
-        if (nodeLock == null) {
-          synchronized (ResourceUtils.class) {
-            Map<String, ResourceInformation> nodeResources =
-                initializeNodeResourceInformation(conf);
-            addManadtoryResources(nodeResources);
-            checkMandatatoryResources(nodeResources);
-            readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
-            nodeLock = new Object();
-          }
-        }
-      }
-    }
-    return readOnlyNodeResources;
-  }
-
-  private static Map<String, ResourceInformation>
-  initializeNodeResourceInformation(Configuration conf) {
-    Map<String, ResourceInformation> nodeResources = new HashMap<>();
-    try {
-      addResourcesFileToConf(
-          YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
-      for (Map.Entry<String, String> entry : conf) {
-        String key = entry.getKey();
-        String value = entry.getValue();
-        if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
-          addResourceInformation(key, value, nodeResources);
-        }
-      }
-    } catch (FileNotFoundException fe) {
-      LOG.info("Couldn't find node resources file");
-    }
-    return nodeResources;
-  }
-
-  private static void addResourceInformation(String prop, String value,
-      Map<String, ResourceInformation> nodeResources) {
-    String[] parts = prop.split("\\.");
-    LOG.info("Found resource entry " + prop);
-    if (parts.length == 4) {
-      String resourceType = parts[3];
-      if (!nodeResources.containsKey(resourceType)) {
-        nodeResources
-            .put(resourceType, ResourceInformation.newInstance(resourceType));
-      }
-      String units = getUnits(value);
-      Long resourceValue =
-          Long.valueOf(value.substring(0, value.length() - units.length()));
-      nodeResources.get(resourceType).setValue(resourceValue);
-      nodeResources.get(resourceType).setUnits(units);
-      LOG.debug("Setting value for resource type " + resourceType + " to "
-              + resourceValue + " with units " + units);
-    }
-  }
-
-  @VisibleForTesting
-  synchronized public static void resetNodeResources() {
-    nodeLock = null;
-  }
-
-  public static Resource getResourceTypesMinimumAllocation() {
-    Map<String, ResourceInformation> resourceTypes = getResourceTypes();
-    Resource ret = Resource.newInstance(0, 0);
-    for (Map.Entry<String, ResourceInformation> entry : resourceTypes
-        .entrySet()) {
-      String name = entry.getKey();
-      if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
-        ret.setMemorySize(entry.getValue().getMinimumAllocation());
-        continue;
-      }
-      if (name.equals(ResourceInformation.VCORES.getName())) {
-        Long tmp = entry.getValue().getMinimumAllocation();
-        if (tmp > Integer.MAX_VALUE) {
-          tmp = (long) Integer.MAX_VALUE;
-        }
-        ret.setVirtualCores(tmp.intValue());
-        continue;
-      }
-      ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
-    }
-    return ret;
-  }
-
-  /**
-   * Get a Resource object with for the maximum allocation possible.
-   * @return a Resource object with the maximum allocation for the scheduler
-   */
-  public static Resource getResourceTypesMaximumAllocation() {
-    Map<String, ResourceInformation> resourceTypes = getResourceTypes();
-    Resource ret = Resource.newInstance(0, 0);
-    for (Map.Entry<String, ResourceInformation> entry : resourceTypes
-        .entrySet()) {
-      String name = entry.getKey();
-      if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
-        ret.setMemorySize(entry.getValue().getMaximumAllocation());
-        continue;
-      }
-      if (name.equals(ResourceInformation.VCORES.getName())) {
-        Long tmp = entry.getValue().getMaximumAllocation();
-        if (tmp > Integer.MAX_VALUE) {
-          tmp = (long) Integer.MAX_VALUE;
-        }
-        ret.setVirtualCores(tmp.intValue());
-        continue;
-      }
-      ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
-    }
-    return ret;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index d143e93..f62114d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -18,30 +18,31 @@
 
 package org.apache.hadoop.yarn.util.resource;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Arrays;
 
 @InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" })
 @Unstable
 public class Resources {
 
+  private static final Log LOG =
+      LogFactory.getLog(Resources.class);
+
   /**
    * Helper class to create a resource with a fixed value for all resource
    * types. For example, a NONE resource which returns 0 for any resource type.
    */
-  static class FixedValueResource extends Resource {
+  static class FixedValueResource extends BaseResource {
 
-    private Map<String, ResourceInformation> resources;
     private long resourceValue;
     private String name;
 
@@ -53,7 +54,7 @@ public class Resources {
     FixedValueResource(String rName, long value) {
       this.resourceValue = value;
       this.name = rName;
-      resources = initResourceMap();
+      initResourceMap();
     }
 
     private int resourceValueToInt() {
@@ -96,31 +97,6 @@ public class Resources {
     }
 
     @Override
-    public Map<String, ResourceInformation> getResources() {
-      return Collections.unmodifiableMap(this.resources);
-    }
-
-    @Override
-    public ResourceInformation getResourceInformation(String resource)
-        throws YarnException {
-      if (resources.containsKey(resource)) {
-        ResourceInformation value = this.resources.get(resource);
-        ResourceInformation ret = ResourceInformation.newInstance(value);
-        ret.setValue(resourceValue);
-        return ret;
-      }
-      throw new YarnException("" + resource + " not found");
-    }
-
-    @Override
-    public Long getResourceValue(String resource) throws YarnException {
-      if (resources.containsKey(resource)) {
-        return resourceValue;
-      }
-      throw new YarnException("" + resource + " not found");
-    }
-
-    @Override
     public void setResourceInformation(String resource,
         ResourceInformation resourceInformation)
         throws ResourceNotFoundException {
@@ -133,24 +109,24 @@ public class Resources {
       throw new RuntimeException(name + " cannot be modified!");
     }
 
-    private Map<String, ResourceInformation> initResourceMap() {
-      Map<String, ResourceInformation> tmp = new HashMap<>();
-      Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+    private void initResourceMap() {
+      ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
       if (types != null) {
-        for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
-          tmp.put(entry.getKey(),
-              ResourceInformation.newInstance(entry.getValue()));
-          tmp.get(entry.getKey()).setValue(resourceValue);
+        resources = new ResourceInformation[types.length];
+        readOnlyResources = new ResourceInformation[types.length];
+        for (int index = 0; index < types.length; index++) {
+          resources[index] = ResourceInformation.newInstance(types[index]);
+          resources[index].setValue(resourceValue);
+
+          // this is a fix for getVirtualCores returning an int
+          if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES
+              .getName().equals(resources[index].getName())) {
+            resources[index].setValue((long) Integer.MAX_VALUE);
+          }
         }
       }
-      // this is a fix for getVirtualCores returning an int
-      if (resourceValue > Integer.MAX_VALUE) {
-        tmp.get(ResourceInformation.VCORES.getName())
-            .setValue((long) Integer.MAX_VALUE);
-      }
-      return tmp;
+      readOnlyResources = Arrays.copyOf(resources, resources.length);
     }
-
   }
 
   public static Resource createResource(int memory) {
@@ -197,17 +173,19 @@ public class Resources {
   }
 
   public static Resource addTo(Resource lhs, Resource rhs) {
-    for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
+    for (ResourceInformation entry : lhs.getResources()) {
+      String name = entry.getName();
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(name);
-        ResourceInformation lhsValue = entry.getValue();
-        long convertedRhs = UnitsConversionUtil
-            .convert(rhsValue.getUnits(), lhsValue.getUnits(),
-                rhsValue.getValue());
+        ResourceInformation lhsValue = entry;
+
+        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+            ? rhsValue.getValue()
+            : UnitsConversionUtil.convert(rhsValue.getUnits(),
+                lhsValue.getUnits(), rhsValue.getValue());
         lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
+        LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
       }
     }
@@ -219,17 +197,19 @@ public class Resources {
   }
 
   public static Resource subtractFrom(Resource lhs, Resource rhs) {
-    for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
+    for (ResourceInformation entry : lhs.getResources()) {
+      String name = entry.getName();
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(name);
-        ResourceInformation lhsValue = entry.getValue();
-        long convertedRhs = UnitsConversionUtil
-            .convert(rhsValue.getUnits(), lhsValue.getUnits(),
-                rhsValue.getValue());
+        ResourceInformation lhsValue = entry;
+
+        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+            ? rhsValue.getValue()
+            : UnitsConversionUtil.convert(rhsValue.getUnits(),
+                lhsValue.getUnits(), rhsValue.getValue());
         lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
+        LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
       }
     }
@@ -263,10 +243,9 @@ public class Resources {
   }
 
   public static Resource multiplyTo(Resource lhs, double by) {
-    for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
-      ResourceInformation lhsValue = entry.getValue();
+    for (ResourceInformation entry : lhs.getResources()) {
+      String name = entry.getName();
+      ResourceInformation lhsValue = entry;
       lhs.setResourceValue(name, (long) (lhsValue.getValue() * by));
     }
     return lhs;
@@ -282,17 +261,21 @@ public class Resources {
    */
   public static Resource multiplyAndAddTo(
       Resource lhs, Resource rhs, double by) {
-    for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
+    for (ResourceInformation entry : lhs.getResources()) {
+      String name = entry.getName();
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(name);
-        ResourceInformation lhsValue = entry.getValue();
-        long convertedRhs = (long) (UnitsConversionUtil
-            .convert(rhsValue.getUnits(), lhsValue.getUnits(),
-                rhsValue.getValue()) * by);
+        ResourceInformation lhsValue = entry;
+
+        long convertedRhs = (long) (((rhsValue.getUnits()
+            .equals(lhsValue.getUnits()))
+                ? rhsValue.getValue()
+                : UnitsConversionUtil.convert(rhsValue.getUnits(),
+                    lhsValue.getUnits(), rhsValue.getValue()))
+            * by);
         lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
+        LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
       }
     }
@@ -311,10 +294,9 @@ public class Resources {
   
   public static Resource multiplyAndRoundDown(Resource lhs, double by) {
     Resource out = clone(lhs);
-    for (Map.Entry<String, ResourceInformation> entry : out.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
-      ResourceInformation lhsValue = entry.getValue();
+    for (ResourceInformation entry : out.getResources()) {
+      String name = entry.getName();
+      ResourceInformation lhsValue = entry;
       out.setResourceValue(name, (long) (lhsValue.getValue() * by));
     }
     return out;
@@ -416,19 +398,21 @@ public class Resources {
   }
   
   public static boolean fitsIn(Resource smaller, Resource bigger) {
-    for (Map.Entry<String, ResourceInformation> entry : smaller.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
+    for (ResourceInformation entry : smaller.getResources()) {
+      String name = entry.getName();
       try {
         ResourceInformation rhsValue = bigger.getResourceInformation(name);
-        ResourceInformation lhsValue = entry.getValue();
-        long convertedRhs = UnitsConversionUtil
-            .convert(rhsValue.getUnits(), lhsValue.getUnits(),
-                rhsValue.getValue());
+        ResourceInformation lhsValue = entry;
+
+        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+            ? rhsValue.getValue()
+            : UnitsConversionUtil.convert(rhsValue.getUnits(),
+                lhsValue.getUnits(), rhsValue.getValue());
         if(lhsValue.getValue() > convertedRhs) {
           return false;
         }
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
+        LOG.warn("Resource is missing:" + ye.getMessage());
         return false;
       }
     }
@@ -442,19 +426,21 @@ public class Resources {
   
   public static Resource componentwiseMin(Resource lhs, Resource rhs) {
     Resource ret = createResource(0);
-    for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
+    for (ResourceInformation entry : lhs.getResources()) {
+      String name = entry.getName();
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(name);
-        ResourceInformation lhsValue = entry.getValue();
-        long convertedRhs = UnitsConversionUtil
-            .convert(rhsValue.getUnits(), lhsValue.getUnits(),
-                rhsValue.getValue());
+        ResourceInformation lhsValue = entry;
+
+        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+            ? rhsValue.getValue()
+            : UnitsConversionUtil.convert(rhsValue.getUnits(),
+                lhsValue.getUnits(), rhsValue.getValue());
         ResourceInformation outInfo =
             lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue;
         ret.setResourceInformation(name, outInfo);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
+        LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
       }
     }
@@ -463,19 +449,21 @@ public class Resources {
   
   public static Resource componentwiseMax(Resource lhs, Resource rhs) {
     Resource ret = createResource(0);
-    for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
-        .entrySet()) {
-      String name = entry.getKey();
+    for (ResourceInformation entry : lhs.getResources()) {
+      String name = entry.getName();
       try {
         ResourceInformation rhsValue = rhs.getResourceInformation(name);
-        ResourceInformation lhsValue = entry.getValue();
-        long convertedRhs = UnitsConversionUtil
-            .convert(rhsValue.getUnits(), lhsValue.getUnits(),
-                rhsValue.getValue());
+        ResourceInformation lhsValue = entry;
+
+        long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+            ? rhsValue.getValue()
+            : UnitsConversionUtil.convert(rhsValue.getUnits(),
+                lhsValue.getUnits(), rhsValue.getValue());
         ResourceInformation outInfo =
             lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue;
         ret.setResourceInformation(name, outInfo);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
+        LOG.warn("Resource is missing:" + ye.getMessage());
         continue;
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 38554b6..b530150 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -276,13 +276,17 @@ public class TestResourceUtils {
       String resourceFile = entry.getKey();
       ResourceUtils.resetNodeResources();
       File dest;
-      File source =
-          new File(conf.getClassLoader().getResource(resourceFile).getFile());
+      File source = new File(
+          conf.getClassLoader().getResource(resourceFile).getFile());
       dest = new File(source.getParent(), "node-resources.xml");
       FileUtils.copyFile(source, dest);
-      Map<String, ResourceInformation> actual =
-          ResourceUtils.getNodeResourceInformation(conf);
-      Assert.assertEquals(entry.getValue().getResources(), actual);
+      Map<String, ResourceInformation> actual = ResourceUtils
+          .getNodeResourceInformation(conf);
+      Assert.assertEquals(actual.size(),
+          entry.getValue().getResources().length);
+      for (ResourceInformation resInfo : entry.getValue().getResources()) {
+        Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+      }
       dest.delete();
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 1555e55..a8404fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -105,7 +105,7 @@ public class TestResources {
         unboundedClone.compareTo(createResource(0, Integer.MAX_VALUE)) > 0);
   }
 
-  @Test(timeout=10000)
+  @Test(timeout = 10000)
   public void testCompareToWithNoneResource() {
     assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0);
     assertTrue(Resources.none().compareTo(createResource(1, 0)) < 0);
@@ -114,7 +114,6 @@ public class TestResources {
     assertTrue(Resources.none().compareTo(createResource(1, 0, 0)) < 0);
     assertTrue(Resources.none().compareTo(createResource(0, 1, 0)) < 0);
     assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
-    assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
   }
 
   @Test(timeout=10000)
@@ -246,7 +245,9 @@ public class TestResources {
   }
 
   @Test
-  public void testMultiplyAndAddTo() {
+  public void testMultiplyAndAddTo() throws Exception {
+    unsetExtraResourceType();
+    setupExtraResourceType();
     assertEquals(createResource(6, 4),
         multiplyAndAddTo(createResource(3, 1), createResource(2, 2), 1.5));
     assertEquals(createResource(6, 4, 0),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 8ef6d99..0982ef9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -178,16 +178,15 @@ public class RMAppAttemptMetrics {
 
   private void updateUsageMap(Resource allocated, long deltaUsedMillis,
       Map<String, AtomicLong> targetMap) {
-    for (Map.Entry<String, ResourceInformation> entry : allocated.getResources()
-        .entrySet()) {
+    for (ResourceInformation entry : allocated.getResources()) {
       AtomicLong resourceUsed;
-      if (!targetMap.containsKey(entry.getKey())) {
+      if (!targetMap.containsKey(entry.getName())) {
         resourceUsed = new AtomicLong(0);
-        targetMap.put(entry.getKey(), resourceUsed);
+        targetMap.put(entry.getName(), resourceUsed);
 
       }
-      resourceUsed = targetMap.get(entry.getKey());
-      resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis)
+      resourceUsed = targetMap.get(entry.getName());
+      resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis)
           / DateUtils.MILLIS_PER_SECOND);
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index d7bf1bc..db63cd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -1006,13 +1006,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity
{
       for (RMContainer rmContainer : this.liveContainers.values()) {
         long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
         Resource resource = rmContainer.getContainer().getResource();
-        for (Map.Entry<String, ResourceInformation> entry : resource
-            .getResources().entrySet()) {
+        for (ResourceInformation entry : resource.getResources()) {
           long value = RMServerUtils
-              .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
-          value += entry.getValue().getValue() * usedMillis
+              .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
+          value += entry.getValue() * usedMillis
               / DateUtils.MILLIS_PER_SECOND;
-          resourceSecondsMap.put(entry.getKey(), value);
+          resourceSecondsMap.put(entry.getName(), value);
         }
       }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 887b854..81491b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
+import java.util.Arrays;
 import java.util.EnumSet;
 
 import javax.xml.bind.annotation.XmlRootElement;
@@ -73,7 +74,7 @@ public class SchedulerInfo {
   }
 
   public String getSchedulerResourceTypes() {
-    return minAllocResource.getResource().getResources().keySet().toString();
+    return Arrays.toString(minAllocResource.getResource().getResources());
   }
 
   public int getMaxClusterLevelAppPriority() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/afd301ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 29e6691..8179321 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -249,6 +249,7 @@ public class TestAppManager{
     ResourceScheduler scheduler = mockResourceScheduler();
     ((RMContextImpl)rmContext).setScheduler(scheduler);
     Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     ((RMContextImpl) rmContext).setYarnConfiguration(conf);
     ApplicationMasterService masterService =
         new ApplicationMasterService(rmContext, scheduler);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message