hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject hadoop git commit: YARN-5342. Improve non-exclusive node partition resource allocation in Capacity Scheduler. (Sunil G via wangda)
Date Wed, 27 Jul 2016 04:20:04 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0f6ff2065 -> 55fa19d57


YARN-5342. Improve non-exclusive node partition resource allocation in Capacity Scheduler.
(Sunil G via wangda)

(cherry picked from commit 49969b16cdba0f251b9f8bf3d8df9906e38b5c61)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55fa19d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55fa19d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55fa19d5

Branch: refs/heads/branch-2
Commit: 55fa19d576e1aa96fe4c2321550b8be9036c556a
Parents: 0f6ff20
Author: Wangda Tan <wangda@apache.org>
Authored: Tue Jul 26 18:14:09 2016 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Tue Jul 26 21:16:12 2016 -0700

----------------------------------------------------------------------
 .../allocator/RegularContainerAllocator.java         | 15 +++++++++++----
 .../scheduler/capacity/TestApplicationPriority.java  |  2 ++
 .../capacity/TestNodeLabelContainerAllocation.java   | 13 ++++++-------
 3 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fa19d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 4bae5be..29b37d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -695,15 +696,21 @@ public class RegularContainerAllocator extends AbstractContainerAllocator
{
       }
 
       // Non-exclusive scheduling opportunity is different: we need reset
-      // it every time to make sure non-labeled resource request will be
+      // it when:
+      // - It allocated on the default partition
+      //
+      // This is to make sure non-labeled resource request will be
       // most likely allocated on non-labeled nodes first.
-      application.resetMissedNonPartitionedRequestSchedulingOpportunity(
-          schedulerKey);
+      if (StringUtils.equals(node.getPartition(),
+          RMNodeLabelsManager.NO_LABEL)) {
+        application
+            .resetMissedNonPartitionedRequestSchedulingOpportunity(schedulerKey);
+      }
     }
 
     return allocationResult;
   }
-  
+
   private ContainerAllocation allocate(Resource clusterResource,
       FiCaSchedulerNode node, SchedulingMode schedulingMode,
       ResourceLimits resourceLimits, SchedulerRequestKey schedulerKey,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fa19d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index 9bb8827..d862c75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
@@ -280,6 +281,7 @@ public class TestApplicationPriority {
     // If app3 (highest priority among rest) gets active, it indicates that
     // priority is working with pendingApplications.
     rm.killApp(app1.getApplicationId());
+    rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED);
 
     // kick the scheduler, app3 (high among pending) gets free space
     MockAM am3 = MockRM.launchAM(app3, rm, nm1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fa19d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index df2c9ff..9070577 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -768,8 +768,6 @@ public class TestNodeLabelContainerAllocation {
     rm1.start();
     MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // label = y
     MockNM nm2 = rm1.registerNode("h2:1234", 100 * GB); // label = <empty>
-    
-    ContainerId nextContainerId;
 
     // launch an app to queue b1 (label = y), AM container should be launched in nm3
     RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1");
@@ -777,12 +775,13 @@ public class TestNodeLabelContainerAllocation {
     
     // request containers from am2, priority=1 asks for "" and priority=2 asks
     // for "y", "y" container should be allocated first
-    nextContainerId =
-        ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
     am1.allocate("*", 1 * GB, 1, 1, new ArrayList<ContainerId>(), "");
     am1.allocate("*", 1 * GB, 1, 2, new ArrayList<ContainerId>(), "y");
-    Assert.assertTrue(rm1.waitForState(nm1, nextContainerId,
-        RMContainerState.ALLOCATED));
+
+    // Do a node heartbeat once
+    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+    cs.handle(new NodeUpdateSchedulerEvent(
+        rm1.getRMContext().getRMNodes().get(nm1.getNodeId())));
     
     // Check pending resource for am2, priority=1 doesn't get allocated before
     // priority=2 allocated
@@ -1674,7 +1673,7 @@ public class TestNodeLabelContainerAllocation {
     // Test case 7
     // After c allocated, d will go first because it has less used_capacity(x)
     // than c
-    doNMHeartbeat(rm, nm1.getNodeId(), 2);
+    doNMHeartbeat(rm, nm1.getNodeId(), 1);
     checkNumOfContainersInAnAppOnGivenNode(2, nm1.getNodeId(),
         cs.getApplicationAttempt(am1.getApplicationAttemptId()));
     checkNumOfContainersInAnAppOnGivenNode(3, nm1.getNodeId(),


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message