Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: yarn-commits@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
Subject: svn commit: r1489070 [2/2] - in
 /hadoop/common/trunk/hadoop-yarn-project: ./
 hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/
 hadoop-yarn/hadoop-yarn-server/hadoop-y...
Date: Mon, 03 Jun 2013 17:33:56 -0000
To: yarn-commits@hadoop.apache.org
From: tucu@apache.org
Message-Id: <20130603173357.86CD723889E2@eris.apache.org>

Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java?rev=1489070&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestDominantResourceFairnessPolicy.java Mon Jun  3 17:33:55 2013
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies;
+
+import static org.junit.Assert.*;
+
+import java.util.Comparator;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FakeSchedulable;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Test;
+
+/**
+ * comparator.compare(sched1, sched2) < 0 means that sched1 should get a
+ * container before sched2
+ */
+public class TestDominantResourceFairnessPolicy {
+
+  private Comparator<Schedulable> createComparator(int clusterMem,
+      int clusterCpu) {
+    DominantResourceFairnessPolicy policy = new DominantResourceFairnessPolicy();
+    policy.initialize(BuilderUtils.newResource(clusterMem, clusterCpu));
+    return policy.getComparator();
+  }
+  
+  private Schedulable createSchedulable(int memUsage, int cpuUsage) {
+    return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL, 0, 0);
+  }
+  
+  private Schedulable createSchedulable(int memUsage, int cpuUsage,
+      int minMemShare, int minCpuShare) {
+    return createSchedulable(memUsage, cpuUsage, ResourceWeights.NEUTRAL,
+        minMemShare, minCpuShare);
+  }
+  
+  private Schedulable createSchedulable(int memUsage, int cpuUsage,
+      ResourceWeights weights) {
+    return createSchedulable(memUsage, cpuUsage, weights, 0, 0);
+  }
+
+  
+  private Schedulable createSchedulable(int memUsage, int cpuUsage,
+      ResourceWeights weights, int minMemShare, int minCpuShare) {
+    Resource usage = BuilderUtils.newResource(memUsage, cpuUsage);
+    Resource minShare = BuilderUtils.newResource(minMemShare, minCpuShare);
+    return new FakeSchedulable(Resources.none(), minShare, weights,
+        Resources.none(), usage, 0l);
+  }
+  
+  @Test
+  public void testSameDominantResource() {
+    assertTrue(createComparator(8000, 4).compare(
+        createSchedulable(1000, 1),
+        createSchedulable(2000, 1)) < 0);
+  }
+  
+  @Test
+  public void testDifferentDominantResource() {
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(4000, 3),
+        createSchedulable(2000, 5)) < 0);
+  }
+  
+  @Test
+  public void testOneIsNeedy() {
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(2000, 5, 0, 6),
+        createSchedulable(4000, 3, 0, 0)) < 0);
+  }
+  
+  @Test
+  public void testBothAreNeedy() {
+    assertTrue(createComparator(8000, 100).compare(
+        // dominant share is 2000/8000
+        createSchedulable(2000, 5),
+        // dominant share is 4000/8000
+        createSchedulable(4000, 3)) < 0);
+    assertTrue(createComparator(8000, 100).compare(
+        // dominant min share is 2/3
+        createSchedulable(2000, 5, 3000, 6),
+        // dominant min share is 4/5
+        createSchedulable(4000, 3, 5000, 4)) < 0);
+  }
+  
+  @Test
+  public void testEvenWeightsSameDominantResource() {
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(3000, 1, new ResourceWeights(2.0f)),
+        createSchedulable(2000, 1)) < 0);
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(1000, 3, new ResourceWeights(2.0f)),
+        createSchedulable(1000, 2)) < 0);
+  }
+  
+  @Test
+  public void testEvenWeightsDifferentDominantResource() {
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(1000, 3, new ResourceWeights(2.0f)),
+        createSchedulable(2000, 1)) < 0);
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(3000, 1, new ResourceWeights(2.0f)),
+        createSchedulable(1000, 2)) < 0);
+  }
+  
+  @Test
+  public void testUnevenWeightsSameDominantResource() {
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)),
+        createSchedulable(2000, 1)) < 0);
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)),
+        createSchedulable(1000, 2)) < 0);
+  }
+  
+  @Test
+  public void testUnevenWeightsDifferentDominantResource() {
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(1000, 3, new ResourceWeights(1.0f, 2.0f)),
+        createSchedulable(2000, 1)) < 0);
+    assertTrue(createComparator(8000, 8).compare(
+        createSchedulable(3000, 1, new ResourceWeights(2.0f, 1.0f)),
+        createSchedulable(1000, 2)) < 0);
+  }
+  
+  @Test
+  public void testCalculateShares() {
+    Resource used = Resources.createResource(10, 5);
+    Resource capacity = Resources.createResource(100, 10);
+    ResourceType[] resourceOrder = new ResourceType[2];
+    ResourceWeights shares = new ResourceWeights();
+    DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator =
+        new DominantResourceFairnessPolicy.DominantResourceFairnessComparator();
+    comparator.calculateShares(used, capacity, shares, resourceOrder,
+        ResourceWeights.NEUTRAL);
+    
+    assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001);
+    assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001);
+    assertEquals(ResourceType.CPU, resourceOrder[0]);
+    assertEquals(ResourceType.MEMORY, resourceOrder[1]);
+  }
+}

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1489070&r1=1489069&r2=1489070&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon Jun  3 17:33:55 2013
@@ -31,17 +31,18 @@ Hadoop MapReduce Next Generation - Fair 
 * {Introduction}
 
   Fair scheduling is a method of assigning resources to applications such that 
-  all apps get, on average, an equal share of resources over time. 
-  Hadoop NextGen is capable of scheduling multiple resource types, such as 
-  Memory and CPU. Currently only memory is supported, so a "cluster share" is 
-  a proportion of aggregate memory in the cluster. When there is a single app 
-  running, that app uses the entire cluster. When other apps are submitted, 
-  resources that free up are assigned to the new apps, so that each app gets 
-  roughly the same amount of resources. Unlike the default Hadoop scheduler, 
-  which forms a queue of apps, this lets short apps finish in reasonable time
-  while not starving long-lived apps. It is also a reasonable way to share a 
-  cluster between a number of users. Finally, fair sharing can also work with 
-  app priorities - the priorities are used as weights to determine the 
+  all apps get, on average, an equal share of resources over time.
+  Hadoop NextGen is capable of scheduling multiple resource types. By default,
+  the Fair Scheduler bases scheduling fairness decisions only on memory. It
+  can be configured to schedule with both memory and CPU, using the notion
+  of Dominant Resource Fairness developed by Ghodsi et al. When there is a
+  single app running, that app uses the entire cluster. When other apps are
+  submitted, resources that free up are assigned to the new apps, so that each
+  app eventually on gets roughly the same amount of resources. Unlike the default
+  Hadoop scheduler, which forms a queue of apps, this lets short apps finish in
+  reasonable time while not starving long-lived apps. It is also a reasonable way
+  to share a cluster between a number of users. Finally, fair sharing can also
+  work with app priorities - the priorities are used as weights to determine the 
   fraction of total resources that each app should get.
 
   The scheduler organizes apps further into "queues", and shares resources 
@@ -49,9 +50,10 @@ Hadoop MapReduce Next Generation - Fair 
   called âdefaultâ. If an app specifically lists a queue in a container 
   resource request, the request is submitted to that queue. It is also 
   possible to assign queues based on the user name included with the request 
-  through configuration. Within each queue, fair sharing is used to share 
-  capacity between the running apps. queues can also be given weights to share 
-  the cluster non-proportionally in the config file.
+  through configuration. Within each queue, a scheduling policy is used to share
+  resources between the running apps. The default is memory-based fair sharing,
+  but FIFO and multi-resource with Dominant Resource Fairness can also be
+  configured. Queues can be configured with weights to share the cluster non-evenly.
 
   The fair scheduler supports hierarchical queues. All queues descend from a
   queue named "root". Available resources are distributed among the children
@@ -120,14 +122,6 @@ Hadoop MapReduce Next Generation - Fair 
      queues and their properties, in addition to certain policy defaults. This file
      must be in XML format as described in the next section.
 
- * <<<yarn.scheduler.fair.minimum-allocation-mb>>>
-
-    * The smallest container size the scheduler can allocate, in MB of memory.
-
- * <<<yarn.scheduler.fair.maximum-allocation-mb>>>
-
-    * The largest container the scheduler can allocate, in MB of memory.
-
  * <<<yarn.scheduler.fair.user-as-default-queue>>>
 
     * Whether to use the username associated with the allocation as the default 
@@ -183,17 +177,23 @@ Allocation file format
  * <<Queue elements>>, which represent queues. Each may contain the following
      properties:
 
-   * minResources: minimum MB of aggregate memory the queue expects. If a queue
-     demands resources, and its current allocation is below its configured minimum,
-     it will be assigned available resources before any queue that is not in this
-     situation.  If multiple queues are in this situation, resources go to the
-     queue with the smallest ratio between allocation and minimum. Note that it is
-     possible that a queue that is below its minimum may not immediately get up to
-     its minimum when it submits an application, because already-running jobs may
-     be using those resources.
-
-   * maxResources: maximum MB of aggregate memory a queue is allowed.  A queue
-     will never be assigned a container that would put it over this limit.
+   * minResources: minimum resources the queue is entitled to, in the form
+     "X mb, Y vcores". If a queue's minimum share is not satisfied, it will be
+     offered available resources before any other queue under the same parent.
+     Under the single-resource fairness policy, a queue
+     is considered unsatisfied if its memory usage is below its minimum memory
+     share. Under dominant resource fairness, a queue is considered unsatisfied
+     if its usage for its dominant resource with respect to the cluster capacity
+     is below its minimum share for that resource. If multiple queues are
+     unsatisfied in this situation, resources go to the queue with the smallest
+     ratio between relevant resource usage and minimum. Note that it is
+     possible that a queue that is below its minimum may not immediately get up
+     to its minimum when it submits an application, because already-running jobs
+     may be using those resources.
+
+   * maxResources: maximum resources a queue is allowed, in the form
+     "X mb, Y vcores". A queue will never be assigned a container that would
+     put its aggregate usage over this limit.
 
    * maxRunningApps: limit the number of apps from the queue to run at once
 
@@ -232,13 +232,13 @@ Allocation file format
 <?xml version="1.0"?>
 <allocations>
   <queue name="sample_queue">
-    <minResources>10000</minResources>
-    <maxResources>90000</maxResources>
+    <minResources>10000 mb</minResources>
+    <maxResources>90000 mb</maxResources>
     <maxRunningApps>50</maxRunningApps>
     <weight>2.0</weight>
     <schedulingMode>fair</schedulingMode>
     <queue name="sample_sub_queue">
-      <minResources>5000</minResources>
+      <minResources>5000 mb</minResources>
     </queue>
   </queue>
   <user name="sample_user">