hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [29/50] [abbrv] hadoop git commit: YARN-2162. Add ability in Fair Scheduler to optionally configure maxResources in terms of percentage. (Yufei Gu)
Date Sat, 07 Oct 2017 00:06:33 GMT
YARN-2162. Add ability in Fair Scheduler to optionally configure maxResources in terms of percentage. (Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99292adc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99292adc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99292adc

Branch: refs/heads/YARN-5734
Commit: 99292adcefdc6b8f280b8e100605fb39f755c38a
Parents: 25f31d9
Author: Yufei Gu <yufei@apache.org>
Authored: Thu Oct 5 23:51:58 2017 -0700
Committer: Yufei Gu <yufei@apache.org>
Committed: Thu Oct 5 23:51:58 2017 -0700

----------------------------------------------------------------------
 .../scheduler/fair/AllocationConfiguration.java | 44 ++++-------
 .../fair/AllocationFileLoaderService.java       | 47 +++++++-----
 .../scheduler/fair/FSLeafQueue.java             |  6 +-
 .../scheduler/fair/FSParentQueue.java           |  6 +-
 .../resourcemanager/scheduler/fair/FSQueue.java | 33 +++++---
 .../scheduler/fair/FairScheduler.java           |  3 +-
 .../fair/FairSchedulerConfiguration.java        | 54 +++++++++++--
 .../scheduler/fair/QueueManager.java            |  3 +-
 .../fair/TestAllocationFileLoaderService.java   | 22 +++---
 .../scheduler/fair/TestFSLeafQueue.java         |  2 +-
 .../scheduler/fair/TestFSParentQueue.java       |  3 +
 .../scheduler/fair/TestFairScheduler.java       | 15 ++--
 .../fair/TestFairSchedulerConfiguration.java    | 81 ++++++++++++++++----
 .../fair/TestMaxRunningAppsEnforcer.java        |  5 +-
 .../scheduler/fair/TestQueueManager.java        |  5 +-
 .../webapp/dao/TestFairSchedulerQueueInfo.java  |  3 +
 .../src/site/markdown/FairScheduler.md          |  4 +-
 17 files changed, 226 insertions(+), 110 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 7bd6959..3505bca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.security.AccessType;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -40,15 +38,13 @@ import com.google.common.annotations.VisibleForTesting;
 public class AllocationConfiguration extends ReservationSchedulerConfiguration {
   private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
   private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
-  private static final ResourceCalculator RESOURCE_CALCULATOR =
-      new DefaultResourceCalculator();
   // Minimum resource allocation for each queue
   private final Map<String, Resource> minQueueResources;
   // Maximum amount of resources per queue
   @VisibleForTesting
-  final Map<String, Resource> maxQueueResources;
+  final Map<String, ConfigurableResource> maxQueueResources;
   // Maximum amount of resources for each queue's ad hoc children
-  private final Map<String, Resource> maxChildQueueResources;
+  private final Map<String, ConfigurableResource> maxChildQueueResources;
   // Sharing weights for each queue
   private final Map<String, Float> queueWeights;
   
@@ -60,7 +56,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
   final Map<String, Integer> userMaxApps;
   private final int userMaxAppsDefault;
   private final int queueMaxAppsDefault;
-  private final Resource queueMaxResourcesDefault;
+  private final ConfigurableResource queueMaxResourcesDefault;
 
   // Maximum resource share for each leaf queue that can be used to run AMs
   final Map<String, Float> queueMaxAMShares;
@@ -109,14 +105,14 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
   private final Set<String> nonPreemptableQueues;
 
   public AllocationConfiguration(Map<String, Resource> minQueueResources,
-      Map<String, Resource> maxQueueResources,
-      Map<String, Resource> maxChildQueueResources,
+      Map<String, ConfigurableResource> maxQueueResources,
+      Map<String, ConfigurableResource> maxChildQueueResources,
       Map<String, Integer> queueMaxApps,
       Map<String, Integer> userMaxApps,
       Map<String, Float> queueWeights,
       Map<String, Float> queueMaxAMShares, int userMaxAppsDefault,
       int queueMaxAppsDefault,
-      Resource queueMaxResourcesDefault,
+      ConfigurableResource queueMaxResourcesDefault,
       float queueMaxAMShareDefault,
       Map<String, SchedulingPolicy> schedulingPolicies,
       SchedulingPolicy defaultSchedulingPolicy,
@@ -165,7 +161,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
     queueMaxAMShares = new HashMap<>();
     userMaxAppsDefault = Integer.MAX_VALUE;
     queueMaxAppsDefault = Integer.MAX_VALUE;
-    queueMaxResourcesDefault = Resources.unbounded();
+    queueMaxResourcesDefault = new ConfigurableResource(Resources.unbounded());
     queueMaxAMShareDefault = 0.5f;
     queueAcls = new HashMap<>();
     resAcls = new HashMap<>();
@@ -291,26 +287,18 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
 
   /**
    * Get the maximum resource allocation for the given queue. If the max in not
-   * set, return the larger of the min and the default max.
+   * set, return the default max.
    *
    * @param queue the target queue's name
    * @return the max allocation on this queue
    */
   @VisibleForTesting
-  Resource getMaxResources(String queue) {
-    Resource maxQueueResource = maxQueueResources.get(queue);
+  ConfigurableResource getMaxResources(String queue) {
+    ConfigurableResource maxQueueResource = maxQueueResources.get(queue);
     if (maxQueueResource == null) {
-      Resource minQueueResource = minQueueResources.get(queue);
-      if (minQueueResource != null &&
-          Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(),
-          minQueueResource, queueMaxResourcesDefault)) {
-        return minQueueResource;
-      } else {
-        return queueMaxResourcesDefault;
-      }
-    } else {
-      return maxQueueResource;
+      maxQueueResource = queueMaxResourcesDefault;
     }
+    return maxQueueResource;
   }
   
   /**
@@ -320,7 +308,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
    * @return the max allocation on this queue or null if not set
    */
   @VisibleForTesting
-  Resource getMaxChildResources(String queue) {
+  ConfigurableResource getMaxChildResources(String queue) {
     return maxChildQueueResources.get(queue);
   }
 
@@ -418,9 +406,9 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
     queue.setMaxChildQueueResource(getMaxChildResources(name));
 
     // Set queue metrics.
-    queue.getMetrics().setMinShare(getMinResources(name));
-    queue.getMetrics().setMaxShare(getMaxResources(name));
-    queue.getMetrics().setMaxApps(getQueueMaxApps(name));
+    queue.getMetrics().setMinShare(queue.getMinShare());
+    queue.getMetrics().setMaxShare(queue.getMaxShare());
+    queue.getMetrics().setMaxApps(queue.getMaxRunningApps());
     queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName());
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 4d918c1..597af94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -226,8 +226,8 @@ public class AllocationFileLoaderService extends AbstractService {
     // Create some temporary hashmaps to hold the new allocs, and we only save
     // them in our fields if we have parsed the entire allocs file successfully.
     Map<String, Resource> minQueueResources = new HashMap<>();
-    Map<String, Resource> maxQueueResources = new HashMap<>();
-    Map<String, Resource> maxChildQueueResources = new HashMap<>();
+    Map<String, ConfigurableResource> maxQueueResources = new HashMap<>();
+    Map<String, ConfigurableResource> maxChildQueueResources = new HashMap<>();
     Map<String, Integer> queueMaxApps = new HashMap<>();
     Map<String, Integer> userMaxApps = new HashMap<>();
     Map<String, Float> queueMaxAMShares = new HashMap<>();
@@ -244,7 +244,8 @@ public class AllocationFileLoaderService extends AbstractService {
     Set<String> nonPreemptableQueues = new HashSet<>();
     int userMaxAppsDefault = Integer.MAX_VALUE;
     int queueMaxAppsDefault = Integer.MAX_VALUE;
-    Resource queueMaxResourcesDefault = Resources.unbounded();
+    ConfigurableResource queueMaxResourcesDefault =
+        new ConfigurableResource(Resources.unbounded());
     float queueMaxAMShareDefault = 0.5f;
     long defaultFairSharePreemptionTimeout = Long.MAX_VALUE;
     long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
@@ -305,7 +306,7 @@ public class AllocationFileLoaderService extends AbstractService {
           }
         } else if ("queueMaxResourcesDefault".equals(element.getTagName())) {
           String text = ((Text)element.getFirstChild()).getData().trim();
-          Resource val =
+          ConfigurableResource val =
               FairSchedulerConfiguration.parseResourceConfigValue(text);
           queueMaxResourcesDefault = val;
         } else if ("userMaxAppsDefault".equals(element.getTagName())) {
@@ -448,8 +449,8 @@ public class AllocationFileLoaderService extends AbstractService {
    */
   private void loadQueue(String parentName, Element element,
       Map<String, Resource> minQueueResources,
-      Map<String, Resource> maxQueueResources,
-      Map<String, Resource> maxChildQueueResources,
+      Map<String, ConfigurableResource> maxQueueResources,
+      Map<String, ConfigurableResource> maxChildQueueResources,
       Map<String, Integer> queueMaxApps,
       Map<String, Integer> userMaxApps,
       Map<String, Float> queueMaxAMShares,
@@ -496,17 +497,17 @@ public class AllocationFileLoaderService extends AbstractService {
       Element field = (Element) fieldNode;
       if ("minResources".equals(field.getTagName())) {
         String text = ((Text)field.getFirstChild()).getData().trim();
-        Resource val =
+        ConfigurableResource val =
             FairSchedulerConfiguration.parseResourceConfigValue(text);
-        minQueueResources.put(queueName, val);
+        minQueueResources.put(queueName, val.getResource());
       } else if ("maxResources".equals(field.getTagName())) {
         String text = ((Text)field.getFirstChild()).getData().trim();
-        Resource val =
+        ConfigurableResource val =
             FairSchedulerConfiguration.parseResourceConfigValue(text);
         maxQueueResources.put(queueName, val);
       } else if ("maxChildResources".equals(field.getTagName())) {
         String text = ((Text)field.getFirstChild()).getData().trim();
-        Resource val =
+        ConfigurableResource val =
             FairSchedulerConfiguration.parseResourceConfigValue(text);
         maxChildQueueResources.put(queueName, val);
       } else if ("maxRunningApps".equals(field.getTagName())) {
@@ -606,14 +607,24 @@ public class AllocationFileLoaderService extends AbstractService {
 
     queueAcls.put(queueName, acls);
     resAcls.put(queueName, racls);
-    if (maxQueueResources.containsKey(queueName) &&
-        minQueueResources.containsKey(queueName)
-        && !Resources.fitsIn(minQueueResources.get(queueName),
-            maxQueueResources.get(queueName))) {
-      LOG.warn(
-          String.format("Queue %s has max resources %s less than "
-              + "min resources %s", queueName, maxQueueResources.get(queueName),
-              minQueueResources.get(queueName)));
+    checkMinAndMaxResource(minQueueResources, maxQueueResources, queueName);
+  }
+
+  private void checkMinAndMaxResource(Map<String, Resource> minResources,
+      Map<String, ConfigurableResource> maxResources, String queueName) {
+
+    ConfigurableResource maxConfigurableResource = maxResources.get(queueName);
+    Resource minResource = minResources.get(queueName);
+
+    if (maxConfigurableResource != null && minResource != null) {
+      Resource maxResource = maxConfigurableResource.getResource();
+
+      // check whether max resource is bigger or equals to min resource when max
+      // resource are absolute values
+      if (maxResource != null && !Resources.fitsIn(minResource, maxResource)) {
+        LOG.warn(String.format("Queue %s has max resources %s less than "
+            + "min resources %s", queueName, maxResource, minResource));
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 89ad1e6..60477d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -332,10 +332,10 @@ public class FSLeafQueue extends FSQueue {
       readLock.unlock();
     }
     // Cap demand to maxShare to limit allocation to maxShare
-    demand = Resources.componentwiseMin(tmpDemand, maxShare);
+    demand = Resources.componentwiseMin(tmpDemand, getMaxShare());
     if (LOG.isDebugEnabled()) {
       LOG.debug("The updated demand for " + getName() + " is " + demand
-          + "; the max is " + maxShare);
+          + "; the max is " + getMaxShare());
       LOG.debug("The updated fairshare for " + getName() + " is "
           + getFairShare());
     }
@@ -611,7 +611,7 @@ public class FSLeafQueue extends FSQueue {
         ", Policy: " + policy.getName() +
         ", FairShare: " + getFairShare() +
         ", SteadyFairShare: " + getSteadyFairShare() +
-        ", MaxShare: " + maxShare +
+        ", MaxShare: " + getMaxShare() +
         ", MinShare: " + minShare +
         ", ResourceUsage: " + getResourceUsage() +
         ", Demand: " + getDemand() +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 5b4e4dc..e42c9f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -150,13 +150,13 @@ public class FSParentQueue extends FSQueue {
         }
       }
       // Cap demand to maxShare to limit allocation to maxShare
-      demand = Resources.componentwiseMin(demand, maxShare);
+      demand = Resources.componentwiseMin(demand, getMaxShare());
     } finally {
       writeLock.unlock();
     }
     if (LOG.isDebugEnabled()) {
       LOG.debug("The updated demand for " + getName() + " is " + demand +
-          "; the max is " + maxShare);
+          "; the max is " + getMaxShare());
     }    
   }
   
@@ -300,7 +300,7 @@ public class FSParentQueue extends FSQueue {
         ", Policy: " + policy.getName() +
         ", FairShare: " + getFairShare() +
         ", SteadyFairShare: " + getSteadyFairShare() +
-        ", MaxShare: " + maxShare +
+        ", MaxShare: " + getMaxShare() +
         ", MinShare: " + minShare +
         ", ResourceUsage: " + getResourceUsage() +
         ", Demand: " + getDemand() +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 8ae3cb6..e0df480 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -71,9 +71,9 @@ public abstract class FSQueue implements Queue, Schedulable {
 
   protected float weights;
   protected Resource minShare;
-  protected Resource maxShare;
+  private ConfigurableResource maxShare;
   protected int maxRunningApps;
-  protected Resource maxChildQueueResource;
+  private ConfigurableResource maxChildQueueResource;
 
   // maxAMShare is a value between 0 and 1.
   protected float maxAMShare;
@@ -105,7 +105,7 @@ public abstract class FSQueue implements Queue, Schedulable {
    *
    * @param recursive whether child queues should be reinitialized recursively
    */
-  public void reinit(boolean recursive) {
+  public final void reinit(boolean recursive) {
     AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
     allocConf.initFSQueue(this);
     updatePreemptionVariables();
@@ -157,26 +157,35 @@ public abstract class FSQueue implements Queue, Schedulable {
     return minShare;
   }
 
-  public void setMaxShare(Resource maxShare){
+  public void setMaxShare(ConfigurableResource maxShare){
     this.maxShare = maxShare;
   }
 
+  @Override
+  public Resource getMaxShare() {
+    Resource maxResource = maxShare.getResource(scheduler.getClusterResource());
+
+    // Max resource should be greater than or equal to min resource
+    Resource result = Resources.componentwiseMax(maxResource, minShare);
+
+    if (!Resources.equals(maxResource, result)) {
+      LOG.warn(String.format("Queue %s has max resources %s less than "
+          + "min resources %s", getName(), maxResource, minShare));
+    }
+    return result;
+  }
+
   public Resource getReservedResource() {
     reservedResource.setMemorySize(metrics.getReservedMB());
     reservedResource.setVirtualCores(metrics.getReservedVirtualCores());
     return reservedResource;
   }
 
-  @Override
-  public Resource getMaxShare() {
-    return maxShare;
-  }
-
-  public void setMaxChildQueueResource(Resource maxChildShare){
+  public void setMaxChildQueueResource(ConfigurableResource maxChildShare){
     this.maxChildQueueResource = maxChildShare;
   }
 
-  public Resource getMaxChildQueueResource() {
+  public ConfigurableResource getMaxChildQueueResource() {
     return maxChildQueueResource;
   }
 
@@ -415,7 +424,7 @@ public abstract class FSQueue implements Queue, Schedulable {
             + " because it has reserved containers.");
       }
       return false;
-    } else if (!Resources.fitsIn(getResourceUsage(), maxShare)) {
+    } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Assigning container failed on node '" + node.getNodeName()
             + " because queue resource usage is larger than MaxShare: "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 9cf2b2e..37f583e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1513,7 +1513,8 @@ public class FairScheduler extends
       if ((queue.getParent() != null) &&
           !configuredLeafQueues.contains(queue.getName()) &&
           !configuredParentQueues.contains(queue.getName())) {
-        Resource max = queue.getParent().getMaxChildQueueResource();
+        ConfigurableResource max = queue.getParent().
+            getMaxChildQueueResource();
 
         if (max != null) {
           queue.setMaxShare(max);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 8e8e37b..960299b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
@@ -287,19 +288,60 @@ public class FairSchedulerConfiguration extends Configuration {
    * 
    * @throws AllocationConfigurationException
    */
-  public static Resource parseResourceConfigValue(String val)
+  public static ConfigurableResource parseResourceConfigValue(String val)
       throws AllocationConfigurationException {
+    ConfigurableResource configurableResource;
     try {
       val = StringUtils.toLowerCase(val);
-      int memory = findResource(val, "mb");
-      int vcores = findResource(val, "vcores");
-      return BuilderUtils.newResource(memory, vcores);
+      if (val.contains("%")) {
+        configurableResource = new ConfigurableResource(
+            getResourcePercentage(val));
+      } else {
+        int memory = findResource(val, "mb");
+        int vcores = findResource(val, "vcores");
+        configurableResource = new ConfigurableResource(
+            BuilderUtils.newResource(memory, vcores));
+      }
     } catch (AllocationConfigurationException ex) {
       throw ex;
     } catch (Exception ex) {
       throw new AllocationConfigurationException(
           "Error reading resource config", ex);
     }
+    return configurableResource;
+  }
+
+  private static double[] getResourcePercentage(
+      String val) throws AllocationConfigurationException {
+    double[] resourcePercentage = new double[ResourceType.values().length];
+    String[] strings = val.split(",");
+    if (strings.length == 1) {
+      double percentage = findPercentage(strings[0], "");
+      for (int i = 0; i < ResourceType.values().length; i++) {
+        resourcePercentage[i] = percentage/100;
+      }
+    } else {
+      resourcePercentage[0] = findPercentage(val, "memory")/100;
+      resourcePercentage[1] = findPercentage(val, "cpu")/100;
+    }
+    return resourcePercentage;
+  }
+
+  private static double findPercentage(String val, String units)
+    throws AllocationConfigurationException {
+    final Pattern pattern =
+        Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units);
+    Matcher matcher = pattern.matcher(val);
+    if (!matcher.find()) {
+      if (units.equals("")) {
+        throw new AllocationConfigurationException("Invalid percentage: " +
+            val);
+      } else {
+        throw new AllocationConfigurationException("Missing resource: " +
+            units);
+      }
+    }
+    return Double.parseDouble(matcher.group(1));
   }
 
   public long getUpdateInterval() {
@@ -307,8 +349,8 @@ public class FairSchedulerConfiguration extends Configuration {
   }
   
   private static int findResource(String val, String units)
-    throws AllocationConfigurationException {
-    Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units);
+      throws AllocationConfigurationException {
+    final Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units);
     Matcher matcher = pattern.matcher(val);
     if (!matcher.find()) {
       throw new AllocationConfigurationException("Missing resource: " + units);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index c08d13e..8734877 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -41,7 +41,6 @@ import org.xml.sax.SAXException;
 import com.google.common.annotations.VisibleForTesting;
 import java.util.Iterator;
 import java.util.Set;
-import org.apache.hadoop.yarn.api.records.Resource;
 
 /**
  * Maintains a list of queues as well as scheduling parameters for each queue,
@@ -330,7 +329,7 @@ public class QueueManager {
         !configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) {
       // For ad hoc queues, set their max reource allocations based on
       // their parents' default child settings.
-      Resource maxChild = parent.getMaxChildQueueResource();
+      ConfigurableResource maxChild = parent.getMaxChildQueueResource();
 
       if (maxChild != null) {
         child.setMaxShare(maxChild);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index c8b9ad8..67b46f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -235,21 +235,21 @@ public class TestAllocationFileLoaderService {
         queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
 
     assertEquals(Resources.createResource(2048, 10),
-        queueConf.getMaxResources("root.queueA"));
+        queueConf.getMaxResources("root.queueA").getResource());
     assertEquals(Resources.createResource(5120, 110),
-        queueConf.getMaxResources("root.queueB"));
-    assertEquals(Resources.createResource(5120, 0),
-        queueConf.getMaxResources("root.queueC"));
+        queueConf.getMaxResources("root.queueB").getResource());
+    assertEquals(Resources.createResource(4096, 100),
+        queueConf.getMaxResources("root.queueC").getResource());
     assertEquals(Resources.createResource(4096, 100),
-        queueConf.getMaxResources("root.queueD"));
+        queueConf.getMaxResources("root.queueD").getResource());
     assertEquals(Resources.createResource(4096, 100),
-        queueConf.getMaxResources("root.queueE"));
+        queueConf.getMaxResources("root.queueE").getResource());
     assertEquals(Resources.createResource(4096, 100),
-        queueConf.getMaxResources("root.queueF"));
+        queueConf.getMaxResources("root.queueF").getResource());
     assertEquals(Resources.createResource(4096, 100),
-        queueConf.getMaxResources("root.queueG"));
+        queueConf.getMaxResources("root.queueG").getResource());
     assertEquals(Resources.createResource(4096, 100),
-        queueConf.getMaxResources("root.queueG.queueH"));
+        queueConf.getMaxResources("root.queueG.queueH").getResource());
 
     assertEquals(Resources.createResource(1024, 0),
         queueConf.getMinResources("root.queueA"));
@@ -279,9 +279,9 @@ public class TestAllocationFileLoaderService {
     assertNull("Max child resources unexpectedly set for queue root.queueE",
         queueConf.getMaxChildResources("root.queueE"));
     assertEquals(Resources.createResource(2048, 64),
-        queueConf.getMaxChildResources("root.queueF"));
+        queueConf.getMaxChildResources("root.queueF").getResource());
     assertEquals(Resources.createResource(2048, 64),
-        queueConf.getMaxChildResources("root.queueG"));
+        queueConf.getMaxChildResources("root.queueG").getResource());
     assertNull("Max child resources unexpectedly set for "
         + "queue root.queueG.queueH",
         queueConf.getMaxChildResources("root.queueG.queueH"));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index b6b3f7e..0bba35d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -81,7 +81,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
 
     String queueName = "root.queue1";
     FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
-    schedulable.setMaxShare(maxResource);
+    schedulable.setMaxShare(new ConfigurableResource(maxResource));
     assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE);
     assertEquals(schedulable.getMetrics().getSchedulingPolicy(),
         SchedulingPolicy.DEFAULT_POLICY.getName());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
index 3ae8f83..671b94b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -42,6 +43,8 @@ public class TestFSParentQueue {
     AllocationConfiguration allocConf = new AllocationConfiguration(conf);
     when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
     when(scheduler.getConf()).thenReturn(conf);
+    when(scheduler.getResourceCalculator()).thenReturn(
+        new DefaultResourceCalculator());
     SystemClock clock = SystemClock.getInstance();
     when(scheduler.getClock()).thenReturn(clock);
     notEmptyQueues = new HashSet<FSQueue>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index d5b1fcc..6b2109d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4611,7 +4611,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     QueueManager queueMgr = scheduler.getQueueManager();
     FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
     FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
-    queue2.setMaxShare(Resource.newInstance(1024, 1));
+    queue2.setMaxShare(
+        new ConfigurableResource(Resource.newInstance(1024, 1)));
 
     ApplicationAttemptId appAttId =
         createSchedulingRequest(1024, 1, "queue1", "user1", 3);
@@ -5209,6 +5210,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     scheduler.reinitialize(conf, resourceManager.getRMContext());
 
     Resource maxResource = Resources.createResource(1024 * 8);
+    ConfigurableResource maxResourceConf =
+        new ConfigurableResource(maxResource);
 
     FSAppAttempt app1 = mock(FSAppAttempt.class);
     Mockito.when(app1.getDemand()).thenReturn(maxResource);
@@ -5220,15 +5223,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
 
     FSLeafQueue aQueue =
         new FSLeafQueue("root.queue1.a", scheduler, queue1);
-    aQueue.setMaxShare(maxResource);
+    aQueue.setMaxShare(maxResourceConf);
     aQueue.addApp(app1, true);
 
     FSLeafQueue bQueue =
         new FSLeafQueue("root.queue1.b", scheduler, queue1);
-    bQueue.setMaxShare(maxResource);
+    bQueue.setMaxShare(maxResourceConf);
     bQueue.addApp(app2, true);
 
-    queue1.setMaxShare(maxResource);
+    queue1.setMaxShare(maxResourceConf);
     queue1.addChildQueue(aQueue);
     queue1.addChildQueue(bQueue);
 
@@ -5266,7 +5269,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
     FSLeafQueue child1 =
         scheduler.getQueueManager().getLeafQueue("parent.child1", false);
     Resource resource = Resource.newInstance(4 * GB, 4);
-    child1.setMaxShare(resource);
+    child1.setMaxShare(new ConfigurableResource(resource));
     FSAppAttempt app = mock(FSAppAttempt.class);
     Mockito.when(app.getDemand()).thenReturn(resource);
     Mockito.when(app.getResourceUsage()).thenReturn(resource);
@@ -5295,7 +5298,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
         childQueueString, child1.dumpState());
     FSParentQueue parent =
         scheduler.getQueueManager().getParentQueue("parent", false);
-    parent.setMaxShare(resource);
+    parent.setMaxShare(new ConfigurableResource(resource));
     parent.updateDemand();
 
     String parentQueueString = "{Name: root.parent,"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 8e7b666..999aaae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -20,11 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
 import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
 import static org.junit.Assert.assertEquals;
 
-import java.io.File;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.junit.Test;
 
@@ -32,23 +28,59 @@ public class TestFairSchedulerConfiguration {
   @Test
   public void testParseResourceConfigValue() throws Exception {
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("2 vcores, 1024 mb"));
+        parseResourceConfigValue("2 vcores, 1024 mb").getResource());
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("1024 mb, 2 vcores").getResource());
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("2vcores,1024mb").getResource());
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("1024mb,2vcores").getResource());
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("1024   mb, 2    vcores").getResource());
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("1024 mb, 2 vcores"));
+        parseResourceConfigValue("1024 Mb, 2 vCores").getResource());
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("2vcores,1024mb"));
+        parseResourceConfigValue("  1024 mb, 2 vcores  ").getResource());
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("1024mb,2vcores"));
+        parseResourceConfigValue("  1024.3 mb, 2.35 vcores  ").getResource());
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("1024   mb, 2    vcores"));
+        parseResourceConfigValue("  1024. mb, 2. vcores  ").getResource());
+
+    Resource clusterResource = BuilderUtils.newResource(2048, 4);
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("50% memory, 50% cpu").
+            getResource(clusterResource));
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("50% Memory, 50% CpU").
+            getResource(clusterResource));
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("1024 Mb, 2 vCores"));
+        parseResourceConfigValue("50%").getResource(clusterResource));
+    assertEquals(BuilderUtils.newResource(1024, 4),
+        parseResourceConfigValue("50% memory, 100% cpu").
+        getResource(clusterResource));
+    assertEquals(BuilderUtils.newResource(1024, 4),
+        parseResourceConfigValue(" 100% cpu, 50% memory").
+        getResource(clusterResource));
+    assertEquals(BuilderUtils.newResource(1024, 0),
+        parseResourceConfigValue("50% memory, 0% cpu").
+            getResource(clusterResource));
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("  1024 mb, 2 vcores  "));
+        parseResourceConfigValue("50 % memory, 50 % cpu").
+            getResource(clusterResource));
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("  1024.3 mb, 2.35 vcores  "));
+        parseResourceConfigValue("50%memory,50%cpu").
+            getResource(clusterResource));
     assertEquals(BuilderUtils.newResource(1024, 2),
-        parseResourceConfigValue("  1024. mb, 2. vcores  "));
+        parseResourceConfigValue("  50  %  memory,  50  %  cpu  ").
+            getResource(clusterResource));
+    assertEquals(BuilderUtils.newResource(1024, 2),
+        parseResourceConfigValue("50.% memory, 50.% cpu").
+            getResource(clusterResource));
+
+    clusterResource =  BuilderUtils.newResource(1024 * 10, 4);
+    assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2),
+        parseResourceConfigValue("10.9% memory, 50.6% cpu").
+            getResource(clusterResource));
   }
   
   @Test(expected = AllocationConfigurationException.class)
@@ -70,5 +102,24 @@ public class TestFairSchedulerConfiguration {
   public void testGibberish() throws Exception {
     parseResourceConfigValue("1o24vc0res");
   }
-  
+
+  @Test(expected = AllocationConfigurationException.class)
+  public void testNoUnitsPercentage() throws Exception {
+    parseResourceConfigValue("95%, 50% memory");
+  }
+
+  @Test(expected = AllocationConfigurationException.class)
+  public void testInvalidNumPercentage() throws Exception {
+    parseResourceConfigValue("95A% cpu, 50% memory");
+  }
+
+  @Test(expected = AllocationConfigurationException.class)
+  public void testCpuPercentageMemoryAbsolute() throws Exception {
+    parseResourceConfigValue("50% cpu, 1024 mb");
+  }
+
+  @Test(expected = AllocationConfigurationException.class)
+  public void testMemoryPercentageCpuAbsolute() throws Exception {
+    parseResourceConfigValue("50% memory, 2 vcores");
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
index f77df1e..964ecf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -55,7 +56,9 @@ public class TestMaxRunningAppsEnforcer {
     AllocationConfiguration allocConf = new AllocationConfiguration(
         conf);
     when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
-    
+    when(scheduler.getResourceCalculator()).thenReturn(
+        new DefaultResourceCalculator());
+
     queueManager = new QueueManager(scheduler);
     queueManager.initialize(conf);
     userMaxApps = allocConf.userMaxApps;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 05ab11c..eb2d402 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,6 +51,8 @@ public class TestQueueManager {
 
     when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
     when(scheduler.getConf()).thenReturn(conf);
+    when(scheduler.getResourceCalculator()).thenReturn(
+        new DefaultResourceCalculator());
 
     SystemClock clock = SystemClock.getInstance();
 
@@ -206,7 +209,7 @@ public class TestQueueManager {
 
     queueManager.updateAllocationConfiguration(allocConf);
     queueManager.getQueue("root.test").setMaxChildQueueResource(
-        Resources.createResource(8192, 256));
+        new ConfigurableResource(Resources.createResource(8192, 256)));
 
     FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF);
     assertNotNull("Leaf queue root.test.childC was not created",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
index 67d7340..83b7e41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
 import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -43,6 +44,8 @@ public class TestFairSchedulerQueueInfo {
     when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
     when(scheduler.getConf()).thenReturn(conf);
     when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1));
+    when(scheduler.getResourceCalculator()).thenReturn(
+        new DefaultResourceCalculator());
     SystemClock clock = SystemClock.getInstance();
     when(scheduler.getClock()).thenReturn(clock);
     QueueManager queueManager = new QueueManager(scheduler);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 2ccd5ff..64c450b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -85,9 +85,9 @@ The allocation file must be in XML format. The format contains five types of ele
 
     * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
 
-    * **maxResources**: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit.
+    * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit.
 
-    * **maxChildResources**: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly.
+    * **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
 
     * **maxRunningApps**: limit the number of apps from the queue to run at once
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message