hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From templ...@apache.org
Subject [21/36] hadoop git commit: YARN-6788. [YARN-3926] Improve performance of resource profile branch (Contributed by Sunil Govindan via Daniel Templeton)
Date Tue, 17 Oct 2017 16:26:33 GMT
YARN-6788. [YARN-3926] Improve performance of resource profile branch
(Contributed by Sunil Govindan via Daniel Templeton)

(cherry picked from commit 3aeaafecb823ef6c175ea5d0d9cb726faacaf32d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86a83163
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86a83163
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86a83163

Branch: refs/heads/resource-types
Commit: 86a8316331923d7aa5271d6591b1d75c49cb4b45
Parents: d5134bd
Author: Daniel Templeton <templedf@apache.org>
Authored: Fri Aug 4 08:42:34 2017 -0700
Committer: Daniel Templeton <templedf@apache.org>
Committed: Tue Oct 17 09:25:45 2017 -0700

----------------------------------------------------------------------
 .../dev-support/findbugs-exclude.xml            |  18 +
 .../hadoop/yarn/api/records/Resource.java       | 234 +++++------
 .../yarn/api/records/ResourceInformation.java   |  13 +-
 .../yarn/api/records/impl/BaseResource.java     | 133 ++++++
 .../yarn/api/records/impl/package-info.java     |  22 +
 .../hadoop/yarn/util/UnitsConversionUtil.java   |   8 +-
 .../yarn/util/resource/ResourceUtils.java       | 401 +++++++++++++++++++
 .../hadoop/yarn/util/resource/package-info.java |  22 +
 .../yarn/api/records/impl/pb/ProtoUtils.java    |   5 +-
 .../api/records/impl/pb/ResourcePBImpl.java     | 110 +++--
 .../resource/DominantResourceCalculator.java    |  67 ++--
 .../yarn/util/resource/ResourceUtils.java       | 329 ---------------
 .../hadoop/yarn/util/resource/Resources.java    | 202 +++++-----
 .../yarn/util/resource/TestResourceUtils.java   |  14 +-
 .../yarn/util/resource/TestResources.java       |   7 +-
 .../rmapp/attempt/RMAppAttemptMetrics.java      |  11 +-
 .../scheduler/SchedulerApplicationAttempt.java  |   9 +-
 .../webapp/dao/SchedulerInfo.java               |   3 +-
 .../server/resourcemanager/TestAppManager.java  |   2 +
 19 files changed, 900 insertions(+), 710 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 6825a36..173ee34 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -599,4 +599,22 @@
     <Bug pattern="IS2_INCONSISTENT_SYNC" />
   </Match>
 
+  <!-- Ignore MS_EXPOSE_REP -->
+    <Match>
+    <Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
+    <Method name="getResourceTypesArray" />
+    <Bug pattern="MS_EXPOSE_REP" />
+  </Match>
+
+  <Match>
+    <Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
+    <Method name="getResourceNamesArray" />
+    <Bug pattern="MS_EXPOSE_REP" />
+  </Match>
+
+  <Match>
+    <Class name="org.apache.hadoop.yarn.api.records.impl.BaseResource" />
+    <Method name="getResources" />
+    <Bug pattern="EI_EXPOSE_REP" />
+  </Match>
 </FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index fb31745..bbd4c87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.api.records;
 
+import java.util.Arrays;
+
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -25,13 +27,10 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.Records;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 
 /**
  * <p><code>Resource</code> models a set of computer resources in the 
@@ -60,97 +59,49 @@ import java.util.Map;
 @Stable
 public abstract class Resource implements Comparable<Resource> {
 
-  private static Resource tmpResource = Records.newRecord(Resource.class);
-
-  private static class SimpleResource extends Resource {
-    private long memory;
-    private long vcores;
-    private Map<String, ResourceInformation> resourceInformationMap;
-
-    SimpleResource(long memory, long vcores) {
-      this.memory = memory;
-      this.vcores = vcores;
-
-    }
-    @Override
-    public int getMemory() {
-      return castToIntSafely(memory);
-    }
-    @Override
-    public void setMemory(int memory) {
-      this.memory = memory;
-    }
-    @Override
-    public long getMemorySize() {
-      return memory;
-    }
-    @Override
-    public void setMemorySize(long memory) {
-      this.memory = memory;
-    }
-    @Override
-    public int getVirtualCores() {
-      return castToIntSafely(vcores);
-    }
-    @Override
-    public void setVirtualCores(int vcores) {
-      this.vcores = vcores;
-    }
-    @Override
-    public Map<String, ResourceInformation> getResources() {
-      if (resourceInformationMap == null) {
-        resourceInformationMap = new HashMap<>();
-        resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(),
-            ResourceInformation.newInstance(ResourceInformation.MEMORY_MB));
-        resourceInformationMap.put(ResourceInformation.VCORES.getName(),
-            ResourceInformation.newInstance(ResourceInformation.VCORES));
-      }
-      resourceInformationMap.get(ResourceInformation.MEMORY_MB.getName())
-          .setValue(this.memory);
-      resourceInformationMap.get(ResourceInformation.VCORES.getName())
-          .setValue(this.vcores);
-      return Collections.unmodifiableMap(resourceInformationMap);
-    }
-  }
+  protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
+  protected static final String VCORES = ResourceInformation.VCORES.getName();
 
   @Public
   @Stable
   public static Resource newInstance(int memory, int vCores) {
-    if (tmpResource.getResources().size() > 2) {
+    if (ResourceUtils.getResourceTypesArray().length > 2) {
       Resource ret = Records.newRecord(Resource.class);
       ret.setMemorySize(memory);
       ret.setVirtualCores(vCores);
       return ret;
     }
-    return new SimpleResource(memory, vCores);
+    return new BaseResource(memory, vCores);
   }
 
   @Public
   @Stable
   public static Resource newInstance(long memory, int vCores) {
-    if (tmpResource.getResources().size() > 2) {
+    if (ResourceUtils.getResourceTypesArray().length > 2) {
       Resource ret = Records.newRecord(Resource.class);
       ret.setMemorySize(memory);
       ret.setVirtualCores(vCores);
       return ret;
     }
-    return new SimpleResource(memory, vCores);
+    return new BaseResource(memory, vCores);
   }
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
   public static Resource newInstance(Resource resource) {
-    Resource ret = Resource.newInstance(0, 0);
-    Resource.copy(resource, ret);
+    Resource ret = Resource.newInstance(resource.getMemorySize(),
+        resource.getVirtualCores());
+    if (ResourceUtils.getResourceTypesArray().length > 2) {
+      Resource.copy(resource, ret);
+    }
     return ret;
   }
 
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
   public static void copy(Resource source, Resource dest) {
-    for (Map.Entry<String, ResourceInformation> entry : source.getResources()
-        .entrySet()) {
-      dest.setResourceInformation(entry.getKey(), entry.getValue());
+    for (ResourceInformation entry : source.getResources()) {
+      dest.setResourceInformation(entry.getName(), entry);
     }
   }
 
@@ -251,25 +202,26 @@ public abstract class Resource implements Comparable<Resource> {
    */
   @Public
   @Evolving
-  public abstract Map<String, ResourceInformation> getResources();
+  public abstract ResourceInformation[] getResources();
 
   /**
    * Get ResourceInformation for a specified resource.
    *
    * @param resource name of the resource
    * @return the ResourceInformation object for the resource
-   * @throws YarnException if the resource can't be found
+   * @throws ResourceNotFoundException if the resource can't be found
    */
   @Public
   @Evolving
   public ResourceInformation getResourceInformation(String resource)
-      throws YarnException {
-    if (getResources().containsKey(resource)) {
-      return getResources().get(resource);
+      throws ResourceNotFoundException {
+    Integer index = ResourceUtils.getResourceTypeIndex().get(resource);
+    ResourceInformation[] resources = getResources();
+    if (index != null) {
+      return resources[index];
     }
-    throw new YarnException(
-        "Unknown resource '" + resource + "'. Known resources are "
-            + getResources().keySet());
+    throw new ResourceNotFoundException("Unknown resource '" + resource
+        + "'. Known resources are " + Arrays.toString(resources));
   }
 
   /**
@@ -278,17 +230,13 @@ public abstract class Resource implements Comparable<Resource> {
    *
    * @param resource name of the resource
    * @return the value for the resource
-   * @throws YarnException if the resource can't be found
+   * @throws ResourceNotFoundException if the resource can't be found
    */
   @Public
   @Evolving
-  public Long getResourceValue(String resource) throws YarnException {
-    if (getResources().containsKey(resource)) {
-      return getResources().get(resource).getValue();
-    }
-    throw new YarnException(
-        "Unknown resource '" + resource + "'. Known resources are "
-            + getResources().keySet());
+  public long getResourceValue(String resource)
+      throws ResourceNotFoundException {
+    return getResourceInformation(resource).getValue();
   }
 
   /**
@@ -301,23 +249,18 @@ public abstract class Resource implements Comparable<Resource> {
   @Public
   @Evolving
   public void setResourceInformation(String resource,
-      ResourceInformation resourceInformation) throws ResourceNotFoundException {
-    if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+      ResourceInformation resourceInformation)
+      throws ResourceNotFoundException {
+    if (resource.equals(MEMORY)) {
       this.setMemorySize(resourceInformation.getValue());
       return;
     }
-    if (resource.equals(ResourceInformation.VCORES.getName())) {
+    if (resource.equals(VCORES)) {
       this.setVirtualCores((int) resourceInformation.getValue());
       return;
     }
-    if (getResources().containsKey(resource)) {
-      ResourceInformation
-          .copy(resourceInformation, getResources().get(resource));
-      return;
-    }
-    throw new ResourceNotFoundException(
-        "Unknown resource '" + resource + "'. Known resources are "
-            + getResources().keySet());
+    ResourceInformation storedResourceInfo = getResourceInformation(resource);
+    ResourceInformation.copy(resourceInformation, storedResourceInfo);
   }
 
   /**
@@ -332,21 +275,17 @@ public abstract class Resource implements Comparable<Resource> {
   @Evolving
   public void setResourceValue(String resource, Long value)
       throws ResourceNotFoundException {
-    if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+    if (resource.equals(MEMORY)) {
       this.setMemorySize(value);
       return;
     }
-    if (resource.equals(ResourceInformation.VCORES.getName())) {
+    if (resource.equals(VCORES)) {
       this.setVirtualCores(value.intValue());
       return;
     }
-    if (getResources().containsKey(resource)) {
-      getResources().get(resource).setValue(value);
-      return;
-    }
-    throw new ResourceNotFoundException(
-        "Unknown resource '" + resource + "'. Known resources are "
-            + getResources().keySet());
+
+    ResourceInformation storedResourceInfo = getResourceInformation(resource);
+    storedResourceInfo.setValue(value);
   }
 
   @Override
@@ -356,13 +295,10 @@ public abstract class Resource implements Comparable<Resource> {
     int result = (int) (939769357
         + getMemorySize()); // prime * result = 939769357 initially
     result = prime * result + getVirtualCores();
-    for (Map.Entry<String, ResourceInformation> entry : getResources()
-        .entrySet()) {
-      if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
-          || entry.getKey().equals(ResourceInformation.VCORES.getName())) {
-        continue;
+    for (ResourceInformation entry : getResources()) {
+      if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) {
+        result = prime * result + entry.hashCode();
       }
-      result = prime * result + entry.getValue().hashCode();
     }
     return result;
   }
@@ -379,11 +315,26 @@ public abstract class Resource implements Comparable<Resource> {
       return false;
     }
     Resource other = (Resource) obj;
-    if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other
-        .getVirtualCores()) {
+    if (getMemorySize() != other.getMemorySize()
+        || getVirtualCores() != other.getVirtualCores()) {
+      return false;
+    }
+
+    ResourceInformation[] myVectors = getResources();
+    ResourceInformation[] otherVectors = other.getResources();
+
+    if (myVectors.length != otherVectors.length) {
       return false;
     }
-    return this.getResources().equals(other.getResources());
+
+    for (int i = 0; i < myVectors.length; i++) {
+      ResourceInformation a = myVectors[i];
+      ResourceInformation b = otherVectors[i];
+      if ((a != b) && ((a == null) || !a.equals(b))) {
+        return false;
+      }
+    }
+    return true;
   }
 
   @Override
@@ -391,21 +342,20 @@ public abstract class Resource implements Comparable<Resource> {
     StringBuilder sb = new StringBuilder();
     sb.append("<memory:").append(getMemorySize()).append(", vCores:")
         .append(getVirtualCores());
-    for (Map.Entry<String, ResourceInformation> entry : getResources()
-        .entrySet()) {
-      if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
-          && entry.getValue().getUnits()
+    for (ResourceInformation entry : getResources()) {
+      if (entry.getName().equals(MEMORY)
+          && entry.getUnits()
           .equals(ResourceInformation.MEMORY_MB.getUnits())) {
         continue;
       }
-      if (entry.getKey().equals(ResourceInformation.VCORES.getName())
-          && entry.getValue().getUnits()
+      if (entry.getName().equals(VCORES)
+          && entry.getUnits()
           .equals(ResourceInformation.VCORES.getUnits())) {
         continue;
       }
-      sb.append(", ").append(entry.getKey()).append(": ")
-          .append(entry.getValue().getValue())
-          .append(entry.getValue().getUnits());
+      sb.append(", ").append(entry.getName()).append(": ")
+          .append(entry.getValue())
+          .append(entry.getUnits());
     }
     sb.append(">");
     return sb.toString();
@@ -413,28 +363,30 @@ public abstract class Resource implements Comparable<Resource> {
 
   @Override
   public int compareTo(Resource other) {
-    Map<String, ResourceInformation> thisResources, otherResources;
-    thisResources = this.getResources();
-    otherResources = other.getResources();
-    long diff = thisResources.size() - otherResources.size();
+    ResourceInformation[] thisResources = this.getResources();
+    ResourceInformation[] otherResources = other.getResources();
+
+    // compare memory and vcores first(in that order) to preserve
+    // existing behaviour
+    long diff = this.getMemorySize() - other.getMemorySize();
     if (diff == 0) {
-      // compare memory and vcores first(in that order) to preserve
-      // existing behaviour
-      if (thisResources.keySet().equals(otherResources.keySet())) {
-        diff = this.getMemorySize() - other.getMemorySize();
-        if (diff == 0) {
-          diff = this.getVirtualCores() - other.getVirtualCores();
-        }
-        if (diff == 0) {
-          for (Map.Entry<String, ResourceInformation> entry : thisResources
-              .entrySet()) {
-            if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
-                || entry.getKey()
-                .equals(ResourceInformation.VCORES.getName())) {
-              continue;
-            }
-            diff =
-                entry.getValue().compareTo(otherResources.get(entry.getKey()));
+      diff = this.getVirtualCores() - other.getVirtualCores();
+    }
+    if (diff == 0) {
+      diff = thisResources.length - otherResources.length;
+      if (diff == 0) {
+        int maxLength = ResourceUtils.getResourceTypesArray().length;
+        for (int i = 0; i < maxLength; i++) {
+          // For memory and vcores, we can skip the loop as it's already
+          // compared.
+          if (i < 2) {
+            continue;
+          }
+
+          ResourceInformation entry = thisResources[i];
+          ResourceInformation otherEntry = otherResources[i];
+          if (entry.getName().equals(otherEntry.getName())) {
+            diff = entry.compareTo(otherEntry);
             if (diff != 0) {
               break;
             }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 76e754b..4a49879 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -195,10 +195,15 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
       return false;
     }
     ResourceInformation r = (ResourceInformation) obj;
-    int cmp =
-        UnitsConversionUtil.compare(this.units, this.value, r.units, r.value);
-    return this.name.equals(r.getName()) && this.resourceType
-        .equals(r.getResourceType()) && (cmp == 0);
+    if (!this.name.equals(r.getName())
+        || !this.resourceType.equals(r.getResourceType())) {
+      return false;
+    }
+    if (this.units.equals(r.units)) {
+      return this.value == r.value;
+    }
+    return (UnitsConversionUtil.compare(this.units, this.value, r.units,
+        r.value) == 0);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
new file mode 100644
index 0000000..83db542
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+import java.util.Arrays;
+
+/**
+ * <p>
+ * <code>BaseResource</code> extends Resource to handle base resources such
+ * as memory and CPU.
+ * TODO: We have a long term plan to use AbstractResource when additional
+ * resource types are to be handled as well.
+ * </p>
+ *
+ * <p>
+ * Currently it models both <em>memory</em> and <em>CPU</em>.
+ * </p>
+ *
+ * <p>
+ * The unit for memory is megabytes. CPU is modeled with virtual cores (vcores),
+ * a unit for expressing parallelism. A node's capacity should be configured
+ * with virtual cores equal to its number of physical cores. A container should
+ * be requested with the number of cores it can saturate, i.e. the average
+ * number of threads it expects to have runnable at a time.
+ * </p>
+ *
+ * <p>
+ * Virtual cores take integer values and thus currently CPU-scheduling is very
+ * coarse. A complementary axis for CPU requests that represents processing
+ * power will likely be added in the future to enable finer-grained resource
+ * configuration.
+ * </p>
+ *
+ * @see Resource
+ */
+@Public
+@Unstable
+public class BaseResource extends Resource {
+
+  private ResourceInformation memoryResInfo;
+  private ResourceInformation vcoresResInfo;
+  protected ResourceInformation[] resources = null;
+  protected ResourceInformation[] readOnlyResources = null;
+
+  protected enum MandatoryResources {
+    MEMORY(0), VCORES(1);
+
+    private final int id;
+
+    MandatoryResources(int id) {
+      this.id = id;
+    }
+
+    public int getId() {
+      return this.id;
+    }
+  }
+
+  public BaseResource() {
+    // Base constructor.
+  }
+
+  public BaseResource(long memory, long vcores) {
+    this.memoryResInfo = ResourceInformation.newInstance(MEMORY,
+        ResourceInformation.MEMORY_MB.getUnits(), memory);
+    this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
+
+    resources = new ResourceInformation[MandatoryResources.values().length];
+    readOnlyResources = new ResourceInformation[MandatoryResources
+        .values().length];
+    resources[MandatoryResources.MEMORY.id] = memoryResInfo;
+    resources[MandatoryResources.VCORES.id] = vcoresResInfo;
+    readOnlyResources = Arrays.copyOf(resources, resources.length);
+  }
+
+  @Override
+  @SuppressWarnings("deprecation")
+  public int getMemory() {
+    return (int) memoryResInfo.getValue();
+  }
+
+  @Override
+  @SuppressWarnings("deprecation")
+  public void setMemory(int memory) {
+    this.memoryResInfo.setValue(memory);
+  }
+
+  @Override
+  public long getMemorySize() {
+    return memoryResInfo.getValue();
+  }
+
+  @Override
+  public void setMemorySize(long memory) {
+    this.memoryResInfo.setValue(memory);
+  }
+
+  @Override
+  public int getVirtualCores() {
+    return (int) vcoresResInfo.getValue();
+  }
+
+  @Override
+  public void setVirtualCores(int vcores) {
+    this.vcoresResInfo.setValue(vcores);
+  }
+
+  @Override
+  public ResourceInformation[] getResources() {
+    return readOnlyResources;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
new file mode 100644
index 0000000..d04e339
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.api.records.impl;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
index c7663de..7b737bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
@@ -186,11 +186,11 @@ public class UnitsConversionUtil {
     if (!KNOWN_UNITS.contains(unitB)) {
       throw new IllegalArgumentException("Unknown unit '" + unitB + "'");
     }
-    Converter unitAC = getConverter(unitA);
-    Converter unitBC = getConverter(unitB);
     if (unitA.equals(unitB)) {
-      return Long.valueOf(valueA).compareTo(valueB);
+      return Long.compare(valueA, valueB);
     }
+    Converter unitAC = getConverter(unitA);
+    Converter unitBC = getConverter(unitB);
     int unitAPos = SORTED_UNITS.indexOf(unitA);
     int unitBPos = SORTED_UNITS.indexOf(unitB);
     try {
@@ -201,7 +201,7 @@ public class UnitsConversionUtil {
       } else {
         tmpA = convert(unitA, unitB, valueA);
       }
-      return Long.valueOf(tmpA).compareTo(tmpB);
+      return Long.compare(tmpA, tmpB);
     } catch (IllegalArgumentException ie) {
       BigInteger tmpA = BigInteger.valueOf(valueA);
       BigInteger tmpB = BigInteger.valueOf(valueB);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
new file mode 100644
index 0000000..1d99c50
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -0,0 +1,401 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util.resource;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Helper class to read the resource-types to be supported by the system.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ResourceUtils {
+
+  public static final String UNITS = ".units";
+  public static final String TYPE = ".type";
+
+  private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
+  private static final String VCORES = ResourceInformation.VCORES.getName();
+
+  private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
+  static {
+    DISALLOWED_NAMES.add("memory");
+    DISALLOWED_NAMES.add(MEMORY);
+    DISALLOWED_NAMES.add(VCORES);
+  }
+
+  private static volatile boolean initializedResources = false;
+  private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
+      new ConcurrentHashMap<String, Integer>();
+  private static volatile Map<String, ResourceInformation> resourceTypes;
+  private static volatile String[] resourceNamesArray;
+  private static volatile ResourceInformation[] resourceTypesArray;
+  private static volatile boolean initializedNodeResources = false;
+  private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
+
+  static final Log LOG = LogFactory.getLog(ResourceUtils.class);
+
+  private ResourceUtils() {
+  }
+
+  private static void checkMandatatoryResources(
+      Map<String, ResourceInformation> resourceInformationMap)
+      throws YarnRuntimeException {
+    if (resourceInformationMap.containsKey(MEMORY)) {
+      ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
+      String memUnits = ResourceInformation.MEMORY_MB.getUnits();
+      ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
+      if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
+          .equals(memType)) {
+        throw new YarnRuntimeException(
+            "Attempt to re-define mandatory resource 'memory-mb'. It can only"
+                + " be of type 'COUNTABLE' and have units 'Mi'.");
+      }
+    }
+
+    if (resourceInformationMap.containsKey(VCORES)) {
+      ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
+      String vcoreUnits = ResourceInformation.VCORES.getUnits();
+      ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
+      if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
+          .getResourceType().equals(vcoreType)) {
+        throw new YarnRuntimeException(
+            "Attempt to re-define mandatory resource 'vcores'. It can only be"
+                + " of type 'COUNTABLE' and have units ''(no units).");
+      }
+    }
+  }
+
+  private static void addManadtoryResources(
+      Map<String, ResourceInformation> res) {
+    ResourceInformation ri;
+    if (!res.containsKey(MEMORY)) {
+      LOG.info("Adding resource type - name = " + MEMORY + ", units = "
+          + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
+          + ResourceTypes.COUNTABLE);
+      ri = ResourceInformation
+          .newInstance(MEMORY,
+              ResourceInformation.MEMORY_MB.getUnits());
+      res.put(MEMORY, ri);
+    }
+    if (!res.containsKey(VCORES)) {
+      LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
+          + ResourceTypes.COUNTABLE);
+      ri =
+          ResourceInformation.newInstance(VCORES);
+      res.put(VCORES, ri);
+    }
+  }
+
+  @VisibleForTesting
+  static void initializeResourcesMap(Configuration conf,
+      Map<String, ResourceInformation> resourceInformationMap) {
+
+    String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
+
+    if (resourceNames != null && resourceNames.length != 0) {
+      for (String resourceName : resourceNames) {
+        String resourceUnits = conf.get(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
+        String resourceTypeName = conf.get(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
+            ResourceTypes.COUNTABLE.toString());
+        if (resourceName == null || resourceName.isEmpty()
+            || resourceUnits == null || resourceTypeName == null) {
+          throw new YarnRuntimeException(
+              "Incomplete configuration for resource type '" + resourceName
+                  + "'. One of name, units or type is configured incorrectly.");
+        }
+        if (DISALLOWED_NAMES.contains(resourceName)) {
+          throw new YarnRuntimeException(
+              "Resource type cannot be named '" + resourceName
+                  + "'. That name is disallowed.");
+        }
+        ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
+        LOG.info("Adding resource type - name = " + resourceName + ", units = "
+            + resourceUnits + ", type = " + resourceTypeName);
+        if (resourceInformationMap.containsKey(resourceName)) {
+          throw new YarnRuntimeException(
+              "Error in config, key '" + resourceName + "' specified twice");
+        }
+        resourceInformationMap.put(resourceName, ResourceInformation
+            .newInstance(resourceName, resourceUnits, 0L, resourceType));
+      }
+    }
+    checkMandatatoryResources(resourceInformationMap);
+    addManadtoryResources(resourceInformationMap);
+    resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
+    updateKnownResources();
+    updateResourceTypeIndex();
+  }
+
+  private static void updateKnownResources() {
+    // Update resource names.
+    resourceNamesArray = new String[resourceTypes.size()];
+    resourceTypesArray = new ResourceInformation[resourceTypes.size()];
+
+    int index = 2;
+    for (ResourceInformation resInfo : resourceTypes.values()) {
+      if (resInfo.getName().equals(MEMORY)) {
+        resourceTypesArray[0] = ResourceInformation
+            .newInstance(resourceTypes.get(MEMORY));
+        resourceNamesArray[0] = MEMORY;
+      } else if (resInfo.getName().equals(VCORES)) {
+        resourceTypesArray[1] = ResourceInformation
+            .newInstance(resourceTypes.get(VCORES));
+        resourceNamesArray[1] = VCORES;
+      } else {
+        resourceTypesArray[index] = ResourceInformation.newInstance(resInfo);
+        resourceNamesArray[index] = resInfo.getName();
+        index++;
+      }
+    }
+  }
+
+  private static void updateResourceTypeIndex() {
+    RESOURCE_NAME_TO_INDEX.clear();
+
+    for (int index = 0; index < resourceTypesArray.length; index++) {
+      ResourceInformation resInfo = resourceTypesArray[index];
+      RESOURCE_NAME_TO_INDEX.put(resInfo.getName(), index);
+    }
+  }
+
+  /**
+   * Get associate index of resource types such memory, cpu etc.
+   * This could help to access each resource types in a resource faster.
+   * @return Index map for all Resource Types.
+   */
+  public static Map<String, Integer> getResourceTypeIndex() {
+    return RESOURCE_NAME_TO_INDEX;
+  }
+
+  /**
+   * Get the resource types to be supported by the system.
+   * @return A map of the resource name to a ResouceInformation object
+   *         which contains details such as the unit.
+   */
+  public static Map<String, ResourceInformation> getResourceTypes() {
+    return getResourceTypes(null,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+  }
+
+  /**
+   * Get resource names array, this is mostly for performance perspective. Never
+   * modify returned array.
+   *
+   * @return resourceNamesArray
+   */
+  public static String[] getResourceNamesArray() {
+    getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    return resourceNamesArray;
+  }
+
+  public static ResourceInformation[] getResourceTypesArray() {
+    getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+    return resourceTypesArray;
+  }
+
+  private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf) {
+    return getResourceTypes(conf,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+  }
+
+  private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf, String resourceFile) {
+    if (!initializedResources) {
+      synchronized (ResourceUtils.class) {
+        if (!initializedResources) {
+          Map<String, ResourceInformation> resources = new HashMap<>();
+          if (conf == null) {
+            conf = new YarnConfiguration();
+          }
+          try {
+            addResourcesFileToConf(resourceFile, conf);
+            LOG.debug("Found " + resourceFile + ", adding to configuration");
+            initializeResourcesMap(conf, resources);
+            initializedResources = true;
+          } catch (FileNotFoundException fe) {
+            LOG.info("Unable to find '" + resourceFile
+                + "'. Falling back to memory and vcores as resources", fe);
+            initializeResourcesMap(conf, resources);
+            initializedResources = true;
+          }
+        }
+      }
+    }
+    return resourceTypes;
+  }
+
+  private static InputStream getConfInputStream(String resourceFile,
+      Configuration conf) throws IOException, YarnException {
+
+    ConfigurationProvider provider =
+        ConfigurationProviderFactory.getConfigurationProvider(conf);
+    InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
+    if (ris == null) {
+      if (conf.getResource(resourceFile) == null) {
+        throw new FileNotFoundException("Unable to find " + resourceFile);
+      }
+      throw new IOException(
+          "Unable to open resource types file '" + resourceFile
+              + "'. Using provider " + provider);
+    }
+    return ris;
+  }
+
+  private static void addResourcesFileToConf(String resourceFile,
+      Configuration conf) throws FileNotFoundException {
+    try {
+      InputStream ris = getConfInputStream(resourceFile, conf);
+      LOG.debug("Found " + resourceFile + ", adding to configuration");
+      conf.addResource(ris);
+    } catch (FileNotFoundException fe) {
+      throw fe;
+    } catch (IOException ie) {
+      LOG.fatal("Exception trying to read resource types configuration '"
+          + resourceFile + "'.", ie);
+      throw new YarnRuntimeException(ie);
+    } catch (YarnException ye) {
+      LOG.fatal("YARN Exception trying to read resource types configuration '"
+          + resourceFile + "'.", ye);
+      throw new YarnRuntimeException(ye);
+    }
+  }
+
+  @VisibleForTesting
+  synchronized static void resetResourceTypes() {
+    initializedResources = false;
+  }
+
+  @VisibleForTesting
+  public static void resetResourceTypes(Configuration conf) {
+    synchronized (ResourceUtils.class) {
+      initializedResources = false;
+    }
+    getResourceTypes(conf);
+  }
+
+  public static String getUnits(String resourceValue) {
+    String units;
+    for (int i = 0; i < resourceValue.length(); i++) {
+      if (Character.isAlphabetic(resourceValue.charAt(i))) {
+        units = resourceValue.substring(i);
+        if (StringUtils.isAlpha(units)) {
+          return units;
+        }
+      }
+    }
+    return "";
+  }
+
+  /**
+   * Function to get the resources for a node. This function will look at the
+   * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to
+   * determine the node resources.
+   *
+   * @param conf configuration file
+   * @return a map to resource name to the ResourceInformation object. The map
+   * is guaranteed to have entries for memory and vcores
+   */
+  public static Map<String, ResourceInformation> getNodeResourceInformation(
+      Configuration conf) {
+    if (!initializedNodeResources) {
+      synchronized (ResourceUtils.class) {
+        if (!initializedNodeResources) {
+          Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
+              conf);
+          addManadtoryResources(nodeResources);
+          checkMandatatoryResources(nodeResources);
+          readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
+          initializedNodeResources = true;
+        }
+      }
+    }
+    return readOnlyNodeResources;
+  }
+
+  private static Map<String, ResourceInformation>
+  initializeNodeResourceInformation(Configuration conf) {
+    Map<String, ResourceInformation> nodeResources = new HashMap<>();
+    try {
+      addResourcesFileToConf(
+          YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
+      for (Map.Entry<String, String> entry : conf) {
+        String key = entry.getKey();
+        String value = entry.getValue();
+        if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
+          addResourceInformation(key, value, nodeResources);
+        }
+      }
+    } catch (FileNotFoundException fe) {
+      LOG.info("Couldn't find node resources file");
+    }
+    return nodeResources;
+  }
+
+  private static void addResourceInformation(String prop, String value,
+      Map<String, ResourceInformation> nodeResources) {
+    String[] parts = prop.split("\\.");
+    LOG.info("Found resource entry " + prop);
+    if (parts.length == 4) {
+      String resourceType = parts[3];
+      if (!nodeResources.containsKey(resourceType)) {
+        nodeResources
+            .put(resourceType, ResourceInformation.newInstance(resourceType));
+      }
+      String units = getUnits(value);
+      Long resourceValue =
+          Long.valueOf(value.substring(0, value.length() - units.length()));
+      nodeResources.get(resourceType).setValue(resourceValue);
+      nodeResources.get(resourceType).setUnits(units);
+      LOG.debug("Setting value for resource type " + resourceType + " to "
+              + resourceValue + " with units " + units);
+    }
+  }
+
+  @VisibleForTesting
+  synchronized public static void resetNodeResources() {
+    initializedNodeResources = false;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
new file mode 100644
index 0000000..01c7eb0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.util.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 16b037e..528cf8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -472,9 +472,8 @@ public class ProtoUtils {
       List<YarnProtos.StringLongMapProto> pList) {
     Resource tmp = Resource.newInstance(0, 0);
     Map<String, Long> ret = new HashMap<>();
-    for (Map.Entry<String, ResourceInformation> entry : tmp.getResources()
-        .entrySet()) {
-      ret.put(entry.getKey(), 0L);
+    for (ResourceInformation entry : tmp.getResources()) {
+      ret.put(entry.getName(), 0L);
     }
     if (pList != null) {
       for (YarnProtos.StringLongMapProto p : pList) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index df42bc6..1641619 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -33,14 +34,13 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.Map;
-import java.util.Collections;
 
 
 @Private
 @Unstable
-public class ResourcePBImpl extends Resource {
+public class ResourcePBImpl extends BaseResource {
 
   private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class);
 
@@ -48,10 +48,6 @@ public class ResourcePBImpl extends Resource {
   ResourceProto.Builder builder = null;
   boolean viaProto = false;
 
-  private Map<String, ResourceInformation> resources;
-  private Map<String, ResourceInformation> readOnlyResources;
-
-
   // call via ProtoUtils.convertToProtoFormat(Resource)
   static ResourceProto getProto(Resource r) {
     final ResourcePBImpl pb;
@@ -72,8 +68,6 @@ public class ResourcePBImpl extends Resource {
   public ResourcePBImpl(ResourceProto proto) {
     this.proto = proto;
     viaProto = true;
-    this.readOnlyResources = null;
-    this.resources = null;
     initResources();
   }
 
@@ -101,11 +95,13 @@ public class ResourcePBImpl extends Resource {
   public long getMemorySize() {
     // memory should always be present
     initResources();
-    ResourceInformation ri =
-        this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
-    return UnitsConversionUtil
-        .convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(),
-            ri.getValue());
+    ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()];
+
+    if (ri.getUnits().isEmpty()) {
+      return ri.getValue();
+    }
+    return UnitsConversionUtil.convert(ri.getUnits(),
+        ResourceInformation.MEMORY_MB.getUnits(), ri.getValue());
   }
 
   @Override
@@ -117,23 +113,20 @@ public class ResourcePBImpl extends Resource {
   @Override
   public void setMemorySize(long memory) {
     maybeInitBuilder();
-    getResourceInformation(ResourceInformation.MEMORY_MB.getName())
-        .setValue(memory);
+    getResourceInformation(MEMORY).setValue(memory);
   }
 
   @Override
   public int getVirtualCores() {
     // vcores should always be present
     initResources();
-    return this.getResourceValue(ResourceInformation.VCORES.getName())
-        .intValue();
+    return (int) resources[MandatoryResources.VCORES.getId()].getValue();
   }
 
   @Override
   public void setVirtualCores(int vCores) {
     maybeInitBuilder();
-    getResourceInformation(ResourceInformation.VCORES.getName())
-        .setValue(vCores);
+    getResourceInformation(VCORES).setValue(vCores);
   }
 
   private void initResources() {
@@ -142,6 +135,7 @@ public class ResourcePBImpl extends Resource {
     }
     ResourceProtoOrBuilder p = viaProto ? proto : builder;
     initResourcesMap();
+    Map<String, Integer> indexMap = ResourceUtils.getResourceTypeIndex();
     for (ResourceInformationProto entry : p.getResourceValueMapList()) {
       ResourceTypes type =
           entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
@@ -150,14 +144,16 @@ public class ResourcePBImpl extends Resource {
       long value = entry.hasValue() ? entry.getValue() : 0L;
       ResourceInformation ri =
           ResourceInformation.newInstance(entry.getKey(), units, value, type);
-      if (resources.containsKey(ri.getName())) {
-        resources.get(ri.getName()).setResourceType(ri.getResourceType());
-        resources.get(ri.getName()).setUnits(ri.getUnits());
-        resources.get(ri.getName()).setValue(value);
-      } else {
+      Integer index = indexMap.get(entry.getKey());
+      if (index == null) {
         LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping");
+      } else {
+        resources[index].setResourceType(ri.getResourceType());
+        resources[index].setUnits(ri.getUnits());
+        resources[index].setValue(value);
       }
     }
+    readOnlyResources = Arrays.copyOf(resources, resources.length);
     this.setMemorySize(p.getMemory());
     this.setVirtualCores(p.getVirtualCores());
   }
@@ -173,79 +169,67 @@ public class ResourcePBImpl extends Resource {
     if (!resource.equals(resourceInformation.getName())) {
       resourceInformation.setName(resource);
     }
-    initResources();
-    if (resources.containsKey(resource)) {
-      ResourceInformation.copy(resourceInformation, resources.get(resource));
-    }
+    ResourceInformation storedResourceInfo = getResourceInformation(resource);
+    ResourceInformation.copy(resourceInformation, storedResourceInfo);
   }
 
   @Override
   public void setResourceValue(String resource, Long value)
       throws ResourceNotFoundException {
     maybeInitBuilder();
-    initResources();
     if (resource == null) {
       throw new IllegalArgumentException("resource type object cannot be null");
     }
-    if (resources == null || (!resources.containsKey(resource))) {
-      throw new ResourceNotFoundException(
-          "Resource " + resource + " not found");
-    }
-    resources.get(resource).setValue(value);
+    getResourceInformation(resource).setValue(value);
   }
 
   @Override
-  public Map<String, ResourceInformation> getResources() {
+  public ResourceInformation[] getResources() {
     initResources();
-    return readOnlyResources;
+    return super.getResources();
   }
 
   @Override
-  public ResourceInformation getResourceInformation(String resource) {
+  public ResourceInformation getResourceInformation(String resource)
+      throws ResourceNotFoundException {
     initResources();
-    if (this.resources.containsKey(resource)) {
-      return this.resources.get(resource);
-    }
-    throw new ResourceNotFoundException("Could not find entry for " + resource);
+    return super.getResourceInformation(resource);
   }
 
   @Override
-  public Long getResourceValue(String resource) {
+  public long getResourceValue(String resource)
+      throws ResourceNotFoundException {
     initResources();
-    if (this.resources.containsKey(resource)) {
-      return this.resources.get(resource).getValue();
-    }
-    throw new ResourceNotFoundException("Could not find entry for " + resource);
+    return super.getResourceValue(resource);
   }
 
   private void initResourcesMap() {
     if (resources == null) {
-      resources = new HashMap<>();
-      Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+      ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
       if (types == null) {
         throw new YarnRuntimeException(
             "Got null return value from ResourceUtils.getResourceTypes()");
       }
-      for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
-        resources.put(entry.getKey(),
-            ResourceInformation.newInstance(entry.getValue()));
+
+      resources = new ResourceInformation[types.length];
+      readOnlyResources = new ResourceInformation[types.length];
+      for (ResourceInformation entry : types) {
+        int index = ResourceUtils.getResourceTypeIndex().get(entry.getName());
+        resources[index] = ResourceInformation.newInstance(entry);
       }
-      readOnlyResources = Collections.unmodifiableMap(resources);
     }
   }
 
   synchronized private void mergeLocalToBuilder() {
     builder.clearResourceValueMap();
-    if (resources != null && !resources.isEmpty()) {
-      for (Map.Entry<String, ResourceInformation> entry :
-          resources.entrySet()) {
-        ResourceInformationProto.Builder e =
-            ResourceInformationProto.newBuilder();
-        e.setKey(entry.getKey());
-        e.setUnits(entry.getValue().getUnits());
-        e.setType(
-            ProtoUtils.converToProtoFormat(entry.getValue().getResourceType()));
-        e.setValue(entry.getValue().getValue());
+    if(resources != null && resources.length != 0) {
+      for (ResourceInformation resInfo : resources) {
+        ResourceInformationProto.Builder e = ResourceInformationProto
+            .newBuilder();
+        e.setKey(resInfo.getName());
+        e.setUnits(resInfo.getUnits());
+        e.setType(ProtoUtils.converToProtoFormat(resInfo.getResourceType()));
+        e.setValue(resInfo.getValue());
         builder.addResourceValueMap(e);
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 79bb03d..ea5c8a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -23,10 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 
-import java.util.Set;
 
 /**
  * A {@link ResourceCalculator} which uses the concept of
@@ -56,10 +55,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
       LogFactory.getLog(DominantResourceCalculator.class);
 
 
-  private Set<String> resourceNames;
+  private String[] resourceNames;
 
   public DominantResourceCalculator() {
-    resourceNames = ResourceUtils.getResourceTypes().keySet();
+    resourceNames = ResourceUtils.getResourceNamesArray();
   }
 
   /**
@@ -88,7 +87,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
         } else if (diff <= -1) {
           rhsGreater = true;
         }
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + rName, ye);
       }
@@ -163,7 +162,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
                 .getValue();
         min = min < tmp ? min : tmp;
         max = max > tmp ? max : tmp;
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -187,7 +186,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
           long tmp = availableResource.getValue() / requiredResourceValue;
           min = min < tmp ? min : tmp;
         }
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -206,14 +205,9 @@ public class DominantResourceCalculator extends ResourceCalculator {
 
   @Override
   public boolean isInvalidDivisor(Resource r) {
-    for (String resource : resourceNames) {
-      try {
-        if (r.getResourceValue(resource).equals(0L)) {
-          return true;
-        }
-      } catch (YarnException ye) {
-        throw new IllegalArgumentException(
-            "Error getting resource value for " + resource, ye);
+    for (ResourceInformation res : r.getResources()) {
+      if (res.getValue() == 0L) {
+        return true;
       }
     }
     return false;
@@ -235,7 +229,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
         float tmp =
             (float) aResourceInformation.getValue() / (float) bResourceValue;
         ratio = ratio > tmp ? ratio : tmp;
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -256,7 +250,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
             ret.getResourceInformation(resource);
         resourceInformation.setValue(
             divideAndCeil(resourceInformation.getValue(), denominator));
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -307,7 +301,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
         }
         tmp.setValue(Math.min(value, maximumValue));
         ret.setResourceInformation(resource, tmp);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -347,7 +341,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
         ResourceInformation
             .copy(rResourceInformation, ret.getResourceInformation(resource));
         ret.getResourceInformation(resource).setValue(value);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -372,28 +366,29 @@ public class DominantResourceCalculator extends ResourceCalculator {
     Resource ret = Resource.newInstance(r);
     for (String resource : resourceNames) {
       try {
-        ResourceInformation rResourceInformation =
-            r.getResourceInformation(resource);
-        ResourceInformation stepFactorResourceInformation =
-            stepFactor.getResourceInformation(resource);
+        ResourceInformation rResourceInformation = r
+            .getResourceInformation(resource);
+        ResourceInformation stepFactorResourceInformation = stepFactor
+            .getResourceInformation(resource);
         ResourceInformation tmp = ret.getResourceInformation(resource);
 
-        Long rValue = rResourceInformation.getValue();
-        Long stepFactorValue = UnitsConversionUtil
-            .convert(stepFactorResourceInformation.getUnits(),
-                rResourceInformation.getUnits(),
-                stepFactorResourceInformation.getValue());
-        Long value;
+        long rValue = rResourceInformation.getValue();
+        long stepFactorValue = UnitsConversionUtil.convert(
+            stepFactorResourceInformation.getUnits(),
+            rResourceInformation.getUnits(),
+            stepFactorResourceInformation.getValue());
+        long value;
         if (stepFactorValue != 0) {
-          value = roundUp ?
-              roundUp((long) Math.ceil(rValue * by), stepFactorValue) :
-              roundDown((long) (rValue * by), stepFactorValue);
+          value = roundUp
+              ? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
+              : roundDown((long) (rValue * by), stepFactorValue);
         } else {
-          value =
-              roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
+          value = roundUp
+              ? (long) Math.ceil(rValue * by)
+              : (long) (rValue * by);
         }
         tmp.setValue(value);
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         throw new IllegalArgumentException(
             "Error getting resource information for " + resource, ye);
       }
@@ -416,7 +411,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
         if(sResourceValue > bResourceInformation.getValue()) {
           return false;
         }
-      } catch (YarnException ye) {
+      } catch (ResourceNotFoundException ye) {
         return false;
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a83163/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
deleted file mode 100644
index 938e462..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util.resource;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.ConfigurationProvider;
-import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Helper class to read the resource-types to be supported by the system.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ResourceUtils {
-
-  public static final String UNITS = ".units";
-  public static final String TYPE = ".type";
-
-  private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
-  private static final String VCORES = ResourceInformation.VCORES.getName();
-
-  private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
-  static {
-    DISALLOWED_NAMES.add("memory");
-    DISALLOWED_NAMES.add(MEMORY);
-    DISALLOWED_NAMES.add(VCORES);
-  }
-
-  private static volatile Object lock;
-  private static Map<String, ResourceInformation> readOnlyResources;
-  private static volatile Object nodeLock;
-  private static Map<String, ResourceInformation> readOnlyNodeResources;
-
-
-  static final Log LOG = LogFactory.getLog(ResourceUtils.class);
-
-  private ResourceUtils() {
-  }
-
-  private static void checkMandatatoryResources(
-      Map<String, ResourceInformation> resourceInformationMap)
-      throws YarnRuntimeException {
-    if (resourceInformationMap.containsKey(MEMORY)) {
-      ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
-      String memUnits = ResourceInformation.MEMORY_MB.getUnits();
-      ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
-      if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
-          .equals(memType)) {
-        throw new YarnRuntimeException(
-            "Attempt to re-define mandatory resource 'memory-mb'. It can only"
-                + " be of type 'COUNTABLE' and have units 'Mi'.");
-      }
-    }
-
-    if (resourceInformationMap.containsKey(VCORES)) {
-      ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
-      String vcoreUnits = ResourceInformation.VCORES.getUnits();
-      ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
-      if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
-          .getResourceType().equals(vcoreType)) {
-        throw new YarnRuntimeException(
-            "Attempt to re-define mandatory resource 'vcores'. It can only be"
-                + " of type 'COUNTABLE' and have units ''(no units).");
-      }
-    }
-  }
-
-  private static void addManadtoryResources(
-      Map<String, ResourceInformation> res) {
-    ResourceInformation ri;
-    if (!res.containsKey(MEMORY)) {
-      LOG.info("Adding resource type - name = " + MEMORY + ", units = "
-          + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
-          + ResourceTypes.COUNTABLE);
-      ri = ResourceInformation
-          .newInstance(MEMORY,
-              ResourceInformation.MEMORY_MB.getUnits());
-      res.put(MEMORY, ri);
-    }
-    if (!res.containsKey(VCORES)) {
-      LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
-          + ResourceTypes.COUNTABLE);
-      ri =
-          ResourceInformation.newInstance(VCORES);
-      res.put(VCORES, ri);
-    }
-  }
-
-  @VisibleForTesting
-  static void initializeResourcesMap(Configuration conf,
-      Map<String, ResourceInformation> resourceInformationMap) {
-
-    String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
-
-    if (resourceNames != null && resourceNames.length != 0) {
-      for (String resourceName : resourceNames) {
-        String resourceUnits = conf.get(
-            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
-        String resourceTypeName = conf.get(
-            YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
-            ResourceTypes.COUNTABLE.toString());
-        if (resourceName == null || resourceName.isEmpty()
-            || resourceUnits == null || resourceTypeName == null) {
-          throw new YarnRuntimeException(
-              "Incomplete configuration for resource type '" + resourceName
-                  + "'. One of name, units or type is configured incorrectly.");
-        }
-        if (DISALLOWED_NAMES.contains(resourceName)) {
-          throw new YarnRuntimeException(
-              "Resource type cannot be named '" + resourceName
-                  + "'. That name is disallowed.");
-        }
-        ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
-        LOG.info("Adding resource type - name = " + resourceName + ", units = "
-            + resourceUnits + ", type = " + resourceTypeName);
-        if (resourceInformationMap.containsKey(resourceName)) {
-          throw new YarnRuntimeException(
-              "Error in config, key '" + resourceName + "' specified twice");
-        }
-        resourceInformationMap.put(resourceName, ResourceInformation
-            .newInstance(resourceName, resourceUnits, 0L, resourceType));
-      }
-    }
-    checkMandatatoryResources(resourceInformationMap);
-    addManadtoryResources(resourceInformationMap);
-    readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
-  }
-
-  /**
-   * Get the resource types to be supported by the system.
-   * @return A map of the resource name to a ResouceInformation object
-   *         which contains details such as the unit.
-   */
-  public static Map<String, ResourceInformation> getResourceTypes() {
-    return getResourceTypes(null,
-        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
-  }
-
-  private static Map<String, ResourceInformation> getResourceTypes(
-      Configuration conf, String resourceFile) {
-    if (lock == null) {
-      synchronized (ResourceUtils.class) {
-        if (lock == null) {
-          synchronized (ResourceUtils.class) {
-            Map<String, ResourceInformation> resources = new HashMap<>();
-            if (conf == null) {
-              conf = new YarnConfiguration();
-            }
-            try {
-              addResourcesFileToConf(resourceFile, conf);
-              LOG.debug("Found " + resourceFile + ", adding to configuration");
-              initializeResourcesMap(conf, resources);
-              lock = new Object();
-            } catch (FileNotFoundException fe) {
-              LOG.info("Unable to find '" + resourceFile
-                  + "'. Falling back to memory and vcores as resources", fe);
-              initializeResourcesMap(conf, resources);
-              lock = new Object();
-            }
-          }
-        }
-      }
-    }
-    return readOnlyResources;
-  }
-
-  private static InputStream getConfInputStream(String resourceFile,
-      Configuration conf) throws IOException, YarnException {
-
-    ConfigurationProvider provider =
-        ConfigurationProviderFactory.getConfigurationProvider(conf);
-    InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
-    if (ris == null) {
-      if (conf.getResource(resourceFile) == null) {
-        throw new FileNotFoundException("Unable to find " + resourceFile);
-      }
-      throw new IOException(
-          "Unable to open resource types file '" + resourceFile
-              + "'. Using provider " + provider);
-    }
-    return ris;
-  }
-
-  private static void addResourcesFileToConf(String resourceFile,
-      Configuration conf) throws FileNotFoundException {
-    try {
-      InputStream ris = getConfInputStream(resourceFile, conf);
-      LOG.debug("Found " + resourceFile + ", adding to configuration");
-      conf.addResource(ris);
-    } catch (FileNotFoundException fe) {
-      throw fe;
-    } catch (IOException ie) {
-      LOG.fatal("Exception trying to read resource types configuration '"
-          + resourceFile + "'.", ie);
-      throw new YarnRuntimeException(ie);
-    } catch (YarnException ye) {
-      LOG.fatal("YARN Exception trying to read resource types configuration '"
-          + resourceFile + "'.", ye);
-      throw new YarnRuntimeException(ye);
-    }
-  }
-
-  @VisibleForTesting
-  static void resetResourceTypes() {
-    lock = null;
-  }
-
-  public static String getUnits(String resourceValue) {
-    String units;
-    for (int i = 0; i < resourceValue.length(); i++) {
-      if (Character.isAlphabetic(resourceValue.charAt(i))) {
-        units = resourceValue.substring(i);
-        if (StringUtils.isAlpha(units)) {
-          return units;
-        }
-      }
-    }
-    return "";
-  }
-
-  /**
-   * Function to get the resources for a node. This function will look at the
-   * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to
-   * determine the node resources.
-   *
-   * @param conf configuration file
-   * @return a map to resource name to the ResourceInformation object. The map
-   * is guaranteed to have entries for memory and vcores
-   */
-  public static Map<String, ResourceInformation> getNodeResourceInformation(
-      Configuration conf) {
-    if (nodeLock == null) {
-      synchronized (ResourceUtils.class) {
-        if (nodeLock == null) {
-          synchronized (ResourceUtils.class) {
-            Map<String, ResourceInformation> nodeResources =
-                initializeNodeResourceInformation(conf);
-            addManadtoryResources(nodeResources);
-            checkMandatatoryResources(nodeResources);
-            readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
-            nodeLock = new Object();
-          }
-        }
-      }
-    }
-    return readOnlyNodeResources;
-  }
-
-  private static Map<String, ResourceInformation>
-  initializeNodeResourceInformation(Configuration conf) {
-    Map<String, ResourceInformation> nodeResources = new HashMap<>();
-    try {
-      addResourcesFileToConf(
-          YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
-      for (Map.Entry<String, String> entry : conf) {
-        String key = entry.getKey();
-        String value = entry.getValue();
-        if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
-          addResourceInformation(key, value, nodeResources);
-        }
-      }
-    } catch (FileNotFoundException fe) {
-      LOG.info("Couldn't find node resources file");
-    }
-    return nodeResources;
-  }
-
-  private static void addResourceInformation(String prop, String value,
-      Map<String, ResourceInformation> nodeResources) {
-    String[] parts = prop.split("\\.");
-    LOG.info("Found resource entry " + prop);
-    if (parts.length == 4) {
-      String resourceType = parts[3];
-      if (!nodeResources.containsKey(resourceType)) {
-        nodeResources
-            .put(resourceType, ResourceInformation.newInstance(resourceType));
-      }
-      String units = getUnits(value);
-      Long resourceValue =
-          Long.valueOf(value.substring(0, value.length() - units.length()));
-      nodeResources.get(resourceType).setValue(resourceValue);
-      nodeResources.get(resourceType).setUnits(units);
-      LOG.debug("Setting value for resource type " + resourceType + " to "
-              + resourceValue + " with units " + units);
-    }
-  }
-
-  @VisibleForTesting
-  synchronized public static void resetNodeResources() {
-    nodeLock = null;
-  }
-
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message