hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vvasu...@apache.org
Subject [49/50] [abbrv] hadoop git commit: YARN-5587. Add support for resource profiles. (vvasudev via asuresh)
Date Wed, 16 Nov 2016 13:47:00 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 9890296..39d772a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -23,8 +23,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
@@ -40,6 +42,7 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   private Priority priority = null;
   private Resource capability = null;
   private ExecutionTypeRequest executionTypeRequest = null;
+  private ProfileCapability profile = null;
   
   
   public ResourceRequestPBImpl() {
@@ -52,7 +55,7 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   }
   
   public ResourceRequestProto getProto() {
-      mergeLocalToProto();
+    mergeLocalToProto();
     proto = viaProto ? proto : builder.build();
     viaProto = true;
     return proto;
@@ -69,6 +72,9 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
       builder.setExecutionTypeRequest(
           ProtoUtils.convertToProtoFormat(this.executionTypeRequest));
     }
+    if (this.profile != null) {
+      builder.setProfile(converToProtoFormat(this.profile));
+    }
   }
 
   private void mergeLocalToProto() {
@@ -229,7 +235,8 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
         + ", Location: " + getResourceName()
         + ", Relax Locality: " + getRelaxLocality()
         + ", Execution Type Request: " + getExecutionTypeRequest()
-        + ", Node Label Expression: " + getNodeLabelExpression() + "}";
+        + ", Node Label Expression: " + getNodeLabelExpression()
+        + ", Resource Profile: " + getProfileCapability() + "}";
   }
 
   @Override
@@ -250,4 +257,34 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
     }
     builder.setNodeLabelExpression(nodeLabelExpression);
   }
+
+  @Override
+  public void setProfileCapability(ProfileCapability profileCapability) {
+    maybeInitBuilder();
+    if (profile == null) {
+      builder.clearProfile();
+    }
+    this.profile = profileCapability;
+  }
+
+  @Override
+  public ProfileCapability getProfileCapability() {
+    if (profile != null) {
+      return profile;
+    }
+    ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+    if (!p.hasProfile()) {
+      return null;
+    }
+    return new ProfileCapabilityPBImpl(p.getProfile());
+  }
+
+  private ProfileCapabilityProto converToProtoFormat(
+      ProfileCapability profileCapability) {
+    ProfileCapabilityPBImpl tmp = new ProfileCapabilityPBImpl();
+    tmp.setProfileName(profileCapability.getProfileName());
+    tmp.setProfileCapabilityOverride(
+        profileCapability.getProfileCapabilityOverride());
+    return tmp.getProto();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 938e462..86cf872 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.ConfigurationProvider;
 import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
@@ -51,6 +52,8 @@ public class ResourceUtils {
 
   public static final String UNITS = ".units";
   public static final String TYPE = ".type";
+  public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
+  public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
 
   private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
   private static final String VCORES = ResourceInformation.VCORES.getName();
@@ -122,6 +125,86 @@ public class ResourceUtils {
     }
   }
 
+  private static void setMinimumAllocationForMandatoryResources(
+      Map<String, ResourceInformation> res, Configuration conf) {
+    String[][] resourceTypesKeys =
+        {
+          { ResourceInformation.MEMORY_MB.getName(),
+            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+            ResourceInformation.MEMORY_MB.getName()
+          },
+          { ResourceInformation.VCORES.getName(),
+            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
+            ResourceInformation.VCORES.getName()
+          }
+        };
+    for (String[] arr : resourceTypesKeys) {
+      String resourceTypesKey =
+          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
+      long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
+      long minimumConf = conf.getLong(arr[1], -1);
+      long minimum;
+      if (minimumResourceTypes != -1) {
+        minimum = minimumResourceTypes;
+        if (minimumConf != -1) {
+          LOG.warn("Using minimum allocation for memory specified in "
+              + "resource-types config file with key "
+              + minimumResourceTypes + ", ignoring minimum specified using "
+              + arr[1]);
+        }
+      } else {
+        minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+      }
+      ResourceInformation ri = res.get(arr[3]);
+      ri.setMinimumAllocation(minimum);
+    }
+  }
+
+  private static void setMaximumAllocationForMandatoryResources(
+      Map<String, ResourceInformation> res, Configuration conf) {
+    String[][] resourceTypesKeys =
+        {
+          {
+            ResourceInformation.MEMORY_MB.getName(),
+            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
+            ResourceInformation.MEMORY_MB.getName()
+          },
+          {
+            ResourceInformation.VCORES.getName(),
+            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+            String.valueOf(
+              YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
+             ResourceInformation.VCORES.getName()
+          }
+        };
+    for (String[] arr : resourceTypesKeys) {
+      String resourceTypesKey =
+          YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
+      long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
+      long maximumConf = conf.getLong(arr[1], -1);
+      long maximum;
+      if (maximumResourceTypes != -1) {
+        maximum = maximumResourceTypes;
+        if (maximumConf != -1) {
+          LOG.warn("Using maximum allocation for memory specified in "
+              + "resource-types config file with key "
+              + maximumResourceTypes + ", ignoring maximum specified using "
+              + arr[1]);
+        }
+      } else {
+        maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+      }
+      ResourceInformation ri = res.get(arr[3]);
+      ri.setMaximumAllocation(maximum);
+    }
+  }
+
   @VisibleForTesting
   static void initializeResourcesMap(Configuration conf,
       Map<String, ResourceInformation> resourceInformationMap) {
@@ -135,6 +218,12 @@ public class ResourceUtils {
         String resourceTypeName = conf.get(
             YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
             ResourceTypes.COUNTABLE.toString());
+        Long minimumAllocation = conf.getLong(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+                + MINIMUM_ALLOCATION, 0L);
+        Long maximumAllocation = conf.getLong(
+            YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+                + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
         if (resourceName == null || resourceName.isEmpty()
             || resourceUnits == null || resourceTypeName == null) {
           throw new YarnRuntimeException(
@@ -154,11 +243,14 @@ public class ResourceUtils {
               "Error in config, key '" + resourceName + "' specified twice");
         }
         resourceInformationMap.put(resourceName, ResourceInformation
-            .newInstance(resourceName, resourceUnits, 0L, resourceType));
+            .newInstance(resourceName, resourceUnits, 0L, resourceType,
+                minimumAllocation, maximumAllocation));
       }
     }
     checkMandatatoryResources(resourceInformationMap);
     addManadtoryResources(resourceInformationMap);
+    setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
+    setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
     readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
   }
 
@@ -173,6 +265,12 @@ public class ResourceUtils {
   }
 
   private static Map<String, ResourceInformation> getResourceTypes(
+      Configuration conf) {
+    return getResourceTypes(conf,
+        YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+  }
+
+  private static Map<String, ResourceInformation> getResourceTypes(
       Configuration conf, String resourceFile) {
     if (lock == null) {
       synchronized (ResourceUtils.class) {
@@ -205,6 +303,12 @@ public class ResourceUtils {
 
     ConfigurationProvider provider =
         ConfigurationProviderFactory.getConfigurationProvider(conf);
+    try {
+      provider.init(conf);
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+
     InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
     if (ris == null) {
       if (conf.getResource(resourceFile) == null) {
@@ -241,6 +345,12 @@ public class ResourceUtils {
     lock = null;
   }
 
+  @VisibleForTesting
+  public static void resetResourceTypes(Configuration conf) {
+    lock = null;
+    getResourceTypes(conf);
+  }
+
   public static String getUnits(String resourceValue) {
     String units;
     for (int i = 0; i < resourceValue.length(); i++) {
@@ -326,4 +436,53 @@ public class ResourceUtils {
     nodeLock = null;
   }
 
+  public static Resource getResourceTypesMinimumAllocation() {
+    Map<String, ResourceInformation> resourceTypes = getResourceTypes();
+    Resource ret = Resource.newInstance(0, 0);
+    for (Map.Entry<String, ResourceInformation> entry : resourceTypes
+        .entrySet()) {
+      String name = entry.getKey();
+      if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+        ret.setMemorySize(entry.getValue().getMinimumAllocation());
+        continue;
+      }
+      if (name.equals(ResourceInformation.VCORES.getName())) {
+        Long tmp = entry.getValue().getMinimumAllocation();
+        if (tmp > Integer.MAX_VALUE) {
+          tmp = (long) Integer.MAX_VALUE;
+        }
+        ret.setVirtualCores(tmp.intValue());
+        continue;
+      }
+      ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
+    }
+    return ret;
+  }
+
+  /**
+   * Get a Resource object with for the maximum allocation possible.
+   * @return a Resource object with the maximum allocation for the scheduler
+   */
+  public static Resource getResourceTypesMaximumAllocation() {
+    Map<String, ResourceInformation> resourceTypes = getResourceTypes();
+    Resource ret = Resource.newInstance(0, 0);
+    for (Map.Entry<String, ResourceInformation> entry : resourceTypes
+        .entrySet()) {
+      String name = entry.getKey();
+      if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+        ret.setMemorySize(entry.getValue().getMaximumAllocation());
+        continue;
+      }
+      if (name.equals(ResourceInformation.VCORES.getName())) {
+        Long tmp = entry.getValue().getMaximumAllocation();
+        if (tmp > Integer.MAX_VALUE) {
+          tmp = (long) Integer.MAX_VALUE;
+        }
+        ret.setVirtualCores(tmp.intValue());
+        continue;
+      }
+      ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
+    }
+    return ret;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 059066b..dc7f11e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -75,12 +75,12 @@ public class Resources {
     }
 
     @Override
+    @SuppressWarnings("deprecation")
     public void setMemory(int memory) {
       throw new RuntimeException(name + " cannot be modified!");
     }
 
     @Override
-    @SuppressWarnings("deprecation")
     public void setMemorySize(long memory) {
       throw new RuntimeException(name + " cannot be modified!");
     }
@@ -189,13 +189,7 @@ public class Resources {
   }
 
   public static Resource clone(Resource res) {
-    Resource ret = Resource.newInstance(0, 0);
-    for (Map.Entry<String, ResourceInformation> entry : res.getResources()
-        .entrySet()) {
-      ret.setResourceInformation(entry.getKey(),
-          ResourceInformation.newInstance(entry.getValue()));
-    }
-    return ret;
+    return Resource.newInstance(res);
   }
 
   public static Resource addTo(Resource lhs, Resource rhs) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 4f952b7..92a6b93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -120,6 +121,7 @@ public class ApplicationMasterService extends AbstractService implements
   private final ConcurrentMap<ApplicationAttemptId, AllocateResponseLock> responseMap
=
       new ConcurrentHashMap<ApplicationAttemptId, AllocateResponseLock>();
   protected final RMContext rmContext;
+  private ResourceProfilesManager resourceProfilesManager;
 
   public ApplicationMasterService(String name, RMContext rmContext,
       YarnScheduler scheduler) {
@@ -127,6 +129,7 @@ public class ApplicationMasterService extends AbstractService implements
     this.amLivelinessMonitor = rmContext.getAMLivelinessMonitor();
     this.rScheduler = scheduler;
     this.rmContext = rmContext;
+    this.resourceProfilesManager = rmContext.getResourceProfilesManager();
   }
 
   public ApplicationMasterService(RMContext rmContext,
@@ -301,6 +304,12 @@ public class ApplicationMasterService extends AbstractService implements
       response.setSchedulerResourceTypes(rScheduler
         .getSchedulingResourceTypes());
 
+      if (getConfig().getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+          YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
+        response
+            .setResourceProfiles(resourceProfilesManager.getResourceProfiles());
+      }
+
       return response;
     }
   }
@@ -455,6 +464,9 @@ public class ApplicationMasterService extends AbstractService implements
       List<ResourceRequest> ask = request.getAskList();
       List<ContainerId> release = request.getReleaseList();
 
+      RMServerUtils.convertProfileToResourceCapability(ask, getConfig(),
+          resourceProfilesManager);
+
       ResourceBlacklistRequest blacklistRequest =
           request.getResourceBlacklistRequest();
       List<String> blacklistAdditions =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 98a34d9..f761e20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -29,6 +29,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
 import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
@@ -58,6 +60,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt
     .RMAppAttemptState;
@@ -81,6 +84,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
  */
 public class RMServerUtils {
 
+  private static final Log LOG_HANDLE = LogFactory.getLog(RMServerUtils.class);
+
   private static final String UPDATE_OUTSTANDING_ERROR =
       "UPDATE_OUTSTANDING_ERROR";
   private static final String INCORRECT_CONTAINER_VERSION_ERROR =
@@ -529,4 +534,49 @@ public class RMServerUtils {
     }
     return newApplicationTimeout;
   }
+
+  public static void convertProfileToResourceCapability(
+      List<ResourceRequest> asks, Configuration conf,
+      ResourceProfilesManager resourceProfilesManager) throws YarnException {
+    boolean profilesEnabled =
+        conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    if (!profilesEnabled) {
+      return;
+    }
+    for (ResourceRequest req : asks) {
+      convertProfileToResourceCapability(req, conf, resourceProfilesManager);
+    }
+  }
+
+  public static void convertProfileToResourceCapability(ResourceRequest ask,
+      Configuration conf, ResourceProfilesManager resourceProfilesManager)
+      throws YarnException {
+
+    if (LOG_HANDLE.isDebugEnabled()) {
+      LOG_HANDLE
+          .debug("Converting profile to resource capability for ask " + ask);
+    }
+
+    boolean profilesEnabled =
+        conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    if (!profilesEnabled) {
+      if (ask.getProfileCapability() != null && !ask.getProfileCapability()
+          .getProfileCapabilityOverride().equals(Resources.none())) {
+        ask.setCapability(
+            ask.getProfileCapability().getProfileCapabilityOverride());
+      }
+    } else {
+      if (ask.getProfileCapability() != null) {
+        ask.setCapability(ProfileCapability
+            .toResource(ask.getProfileCapability(),
+                resourceProfilesManager.getResourceProfiles()));
+      }
+    }
+    if (LOG_HANDLE.isDebugEnabled()) {
+      LOG_HANDLE
+          .debug("Converted profile to resource capability for ask " + ask);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index 4bef333..15479e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -88,6 +88,10 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager
{
     while (iterator.hasNext()) {
       Map.Entry entry = (Map.Entry) iterator.next();
       String key = entry.getKey().toString();
+      if (key.isEmpty()) {
+        throw new IOException(
+            "Name of resource profile cannot be an empty string");
+      }
       if (entry.getValue() instanceof Map) {
         Map value = (Map) entry.getValue();
         // ensure memory and vcores are specified

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 235e659..c0db884 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntit
 import org.apache.hadoop.yarn.server.utils.Lock;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.SettableFuture;
@@ -1022,4 +1023,47 @@ public abstract class AbstractYarnScheduler
     }
   }
 
+
+  /**
+   * Get a Resource object with for the minimum allocation possible. If resource
+   * profiles are enabled then the 'minimum' resource profile will be used. If
+   * they are not enabled, use the minimums specified in the config files.
+   *
+   * @return a Resource object with the minimum allocation for the scheduler
+   */
+  public Resource getMinimumAllocation() {
+    boolean profilesEnabled = getConfig()
+        .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    Resource ret;
+    if (!profilesEnabled) {
+      ret = ResourceUtils.getResourceTypesMinimumAllocation();
+    } else {
+      ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+    }
+    LOG.info("Minimum allocation = " + ret);
+    return ret;
+  }
+
+  /**
+   * Get a Resource object with for the maximum allocation possible. If resource
+   * profiles are enabled then the 'maximum' resource profile will be used. If
+   * they are not enabled, use the maximums specified in the config files.
+   *
+   * @return a Resource object with the maximum allocation for the scheduler
+   */
+
+  public Resource getMaximumAllocation() {
+    boolean profilesEnabled = getConfig()
+        .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+            YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+    Resource ret;
+    if (!profilesEnabled) {
+      ret = ResourceUtils.getResourceTypesMaximumAllocation();
+    } else {
+      ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+    }
+    LOG.info("Maximum allocation = " + ret);
+    return ret;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index e487f69..769a9e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -221,8 +221,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
 
       return Resources.createResource(
           Math.min(configuredMaxAllocation.getMemorySize(), maxNodeMemory),
-          Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores)
-      );
+          Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores));
     } finally {
       readLock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c999e26..c61bfb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -36,10 +36,12 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.AccessType;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -274,6 +276,14 @@ public class SchedulerUtils {
   private static void validateResourceRequest(ResourceRequest resReq,
       Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
       throws InvalidResourceRequestException {
+    try {
+      RMServerUtils.convertProfileToResourceCapability(resReq,
+          rmContext.getYarnConfiguration(),
+          rmContext.getResourceProfilesManager());
+    } catch (YarnException ye) {
+      throw new InvalidResourceRequestException(ye);
+    }
+
     if (resReq.getCapability().getMemorySize() < 0 ||
         resReq.getCapability().getMemorySize() > maximumResource.getMemorySize()) {
       throw new InvalidResourceRequestException("Invalid resource request"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 65a08c6..4c13a67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -319,8 +319,8 @@ public class CapacityScheduler extends
       writeLock.lock();
       this.conf = loadCapacitySchedulerConfiguration(configuration);
       validateConf(this.conf);
-      this.minimumAllocation = this.conf.getMinimumAllocation();
-      initMaximumResourceCapability(this.conf.getMaximumAllocation());
+      this.minimumAllocation = super.getMinimumAllocation();
+      initMaximumResourceCapability(super.getMaximumAllocation());
       this.calculator = this.conf.getResourceCalculator();
       this.usePortForNodeName = this.conf.getUsePortForNodeName();
       this.applications = new ConcurrentHashMap<>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 1d04710..87f36fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1380,8 +1380,8 @@ public class FairScheduler extends
       writeLock.lock();
       this.conf = new FairSchedulerConfiguration(conf);
       validateConf(this.conf);
-      minimumAllocation = this.conf.getMinimumAllocation();
-      initMaximumResourceCapability(this.conf.getMaximumAllocation());
+      minimumAllocation = super.getMinimumAllocation();
+      initMaximumResourceCapability(super.getMaximumAllocation());
       incrAllocation = this.conf.getIncrementAllocation();
       updateReservationThreshold();
       continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 5ccde19..b00f161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -239,17 +239,8 @@ public class FifoScheduler extends
     //Use ConcurrentSkipListMap because applications need to be ordered
     this.applications =
         new ConcurrentSkipListMap<>();
-    this.minimumAllocation =
-        Resources.createResource(conf.getInt(
-            YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
-    initMaximumResourceCapability(
-        Resources.createResource(conf.getInt(
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
-          conf.getInt(
-            YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
-            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
+    this.minimumAllocation = super.getMinimumAllocation();
+    initMaximumResourceCapability(super.getMaximumAllocation());
     this.usePortForNodeName = conf.getBoolean(
         YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
         YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 3861624..bd6469e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -100,6 +100,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSe
 
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
@@ -133,6 +134,7 @@ public class MockRM extends ResourceManager {
   public MockRM(Configuration conf, RMStateStore store,
       boolean useNullRMNodeLabelsManager) {
     super();
+    ResourceUtils.resetResourceTypes(conf);
     this.useNullRMNodeLabelsManager = useNullRMNodeLabelsManager;
     init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
     if(store != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 981b045..bbe1dad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -238,6 +238,7 @@ public class TestAppManager{
     ResourceScheduler scheduler = mockResourceScheduler();
     ((RMContextImpl)rmContext).setScheduler(scheduler);
     Configuration conf = new Configuration();
+    ((RMContextImpl) rmContext).setYarnConfiguration(conf);
     ApplicationMasterService masterService =
         new ApplicationMasterService(rmContext, scheduler);
     appMonitor = new TestRMAppManager(rmContext,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 93befcb..256daba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -494,4 +495,38 @@ public class TestApplicationMasterService {
       Assert.fail("Cannot find RMContainer");
     }
   }
+
+  @Test(timeout = 3000000)
+  public void testResourceProfiles() throws Exception {
+
+    MockRM rm = new MockRM(conf);
+    rm.start();
+    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+    RMApp app1 = rm.submitApp(2048);
+    nm1.nodeHeartbeat(true);
+    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+    RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
+    Assert.assertEquals(0, resp.getResourceProfiles().size());
+    rm.stop();
+    conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+    conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+        "profiles/sample-profiles-1.json");
+    rm = new MockRM(conf);
+    rm.start();
+    nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+    app1 = rm.submitApp(2048);
+    nm1.nodeHeartbeat(true);
+    attempt1 = app1.getCurrentAppAttempt();
+    am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+    resp = am1.registerAppAttempt();
+    Assert.assertEquals(3, resp.getResourceProfiles().size());
+    Assert.assertEquals(Resource.newInstance(1024, 1),
+        resp.getResourceProfiles().get("minimum"));
+    Assert.assertEquals(Resource.newInstance(2048, 2),
+        resp.getResourceProfiles().get("default"));
+    Assert.assertEquals(Resource.newInstance(4096, 4),
+        resp.getResourceProfiles().get("maximum"));
+    rm.stop();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 21d22c3..0439fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -110,6 +110,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
@@ -208,6 +209,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
     conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 
       128);
+    ResourceUtils.resetResourceTypes(conf);
     scheduler.init(conf);
     scheduler.start();
     scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -236,6 +238,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
     conf.setInt(
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+    ResourceUtils.resetResourceTypes(conf);
     scheduler.init(conf);
     scheduler.reinitialize(conf, null);
     Assert.assertEquals(256, scheduler.getMinimumResourceCapability().getMemorySize());
@@ -253,6 +256,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
     conf.setInt(
       FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+    ResourceUtils.resetResourceTypes(conf);
     scheduler.init(conf);
     scheduler.reinitialize(conf, null);
     Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getMemorySize());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b46ec2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 89ebeb1..904c746 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
 import org.apache.hadoop.yarn.server.timeline.TimelineStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
 import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
@@ -247,6 +248,7 @@ public class MiniYARNCluster extends CompositeService {
         YarnConfiguration.DEFAULT_YARN_MINICLUSTER_USE_RPC);
     failoverTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
         YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
+    ResourceUtils.resetResourceTypes(conf);
 
     if (useRpc && !useFixedPorts) {
       throw new YarnRuntimeException("Invalid configuration!" +


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message