hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject hadoop git commit: YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources. (Bibin A Chundatt via wangda)
Date Wed, 22 Jul 2015 18:54:20 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 1b3bceb58 -> 76ec26de8


YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on
total-used-resources. (Bibin A Chundatt via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ec26de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ec26de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ec26de

Branch: refs/heads/trunk
Commit: 76ec26de8099dc48ce3812c595b7ab857a600442
Parents: 1b3bceb
Author: Wangda Tan <wangda@apache.org>
Authored: Wed Jul 22 11:54:02 2015 -0700
Committer: Wangda Tan <wangda@apache.org>
Committed: Wed Jul 22 11:54:02 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../scheduler/capacity/LeafQueue.java           |  8 ++-
 .../TestCapacitySchedulerNodeLabelUpdate.java   | 64 ++++++++++++++++++++
 4 files changed, 74 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5100cdf..f751862 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue is 
     more than 2 level. (Ajith S via wangda)
 
+    YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo 
+    should based on total-used-resources. (Bibin A Chundatt via wangda)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index cf543bd..317e61c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -598,7 +598,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity
{
     AggregateAppResourceUsage runningResourceUsage =
         getRunningAggregateAppResourceUsage();
     Resource usedResourceClone =
-        Resources.clone(attemptResourceUsage.getUsed());
+        Resources.clone(attemptResourceUsage.getAllUsed());
     Resource reservedResourceClone =
         Resources.clone(attemptResourceUsage.getReserved());
     return ApplicationResourceUsageReport.newInstance(liveContainers.size(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 0ce4d68..5c283f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -439,7 +439,7 @@ public class LeafQueue extends AbstractCSQueue {
     for (Map.Entry<String, User> entry : users.entrySet()) {
       User user = entry.getValue();
       usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
-          .getUsed()), user.getActiveApplications(), user
+          .getAllUsed()), user.getActiveApplications(), user
           .getPendingApplications(), Resources.clone(user
           .getConsumedAMResources()), Resources.clone(user
           .getUserResourceLimit())));
@@ -1894,7 +1894,11 @@ public class LeafQueue extends AbstractCSQueue {
     public Resource getUsed() {
       return userResourceUsage.getUsed();
     }
-    
+
+    public Resource getAllUsed() {
+      return userResourceUsage.getAllUsed();
+    }
+
     public Resource getUsed(String label) {
       return userResourceUsage.getUsed(label);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index e60e496..0a701d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -95,6 +97,68 @@ public class TestCapacitySchedulerNodeLabelUpdate {
         .getMemory());
   }
 
+  @Test(timeout = 60000)
+  public void testResourceUsage() throws Exception {
+    // set node -> label
+    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y",
+        "z"));
+
+    // set mapping:
+    // h1 -> x
+    // h2 -> y
+    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+
+    // inject node label manager
+    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+      @Override
+      public RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+    rm.getRMContext().setNodeLabelManager(mgr);
+    rm.start();
+    MockNM nm1 = rm.registerNode("h1:1234", 2048);
+    MockNM nm2 = rm.registerNode("h2:1234", 2048);
+    MockNM nm3 = rm.registerNode("h3:1234", 2048);
+
+    ContainerId containerId;
+    // launch an app to queue a1 (label = x), and check all container will
+    // be allocated in h1
+    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
+    ApplicationResourceUsageReport appResourceUsageReport =
+        rm.getResourceScheduler().getAppResourceUsageReport(
+            am1.getApplicationAttemptId());
+    Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
+        .getMemory());
+    Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
+        .getVirtualCores());
+    // request a container.
+    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
+    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+    rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000);
+    appResourceUsageReport =
+        rm.getResourceScheduler().getAppResourceUsageReport(
+            am1.getApplicationAttemptId());
+    Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
+        .getMemory());
+    Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
+        .getVirtualCores());
+    LeafQueue queue =
+        (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
+            .getQueue("a");
+    ArrayList<UserInfo> users = queue.getUsers();
+    for (UserInfo userInfo : users) {
+      if (userInfo.getUsername().equals("user")) {
+        ResourceInfo resourcesUsed = userInfo.getResourcesUsed();
+        Assert.assertEquals(2048, resourcesUsed.getMemory());
+        Assert.assertEquals(2, resourcesUsed.getvCores());
+      }
+    }
+    rm.stop();
+  }
+
   @Test (timeout = 60000)
   public void testNodeUpdate() throws Exception {
     // set node -> label


Mime
View raw message