hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject [11/50] hadoop git commit: YARN-7575. NPE in scheduler UI when max-capacity is not configured. Contributed by Sunil G.
Date Tue, 12 Dec 2017 23:58:46 GMT
YARN-7575. NPE in scheduler UI when max-capacity is not configured. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db4a4a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db4a4a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db4a4a1

Branch: refs/heads/HDFS-7240
Commit: 4db4a4a165d3936d59f5e67947c6bfbc9c3270a5
Parents: daa1cdd
Author: Sunil G <sunilg@apache.org>
Authored: Wed Nov 29 14:50:54 2017 +0530
Committer: Wangda Tan <wangda@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../webapp/CapacitySchedulerPage.java           |   9 +-
 .../TestAbsoluteResourceConfiguration.java      | 116 +++++++++++++++++++
 2 files changed, 121 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db4a4a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 3f68e07..4933d34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -161,10 +161,11 @@ class CapacitySchedulerPage extends RmView {
           .__("Configured Capacity:",
               capacities.getConfiguredMinResource().toString())
           .__("Configured Max Capacity:",
-              capacities.getConfiguredMaxResource().getResource()
-                  .equals(Resources.none())
-                      ? "unlimited"
-                      : capacities.getConfiguredMaxResource().toString())
+              (capacities.getConfiguredMaxResource() == null
+                  || capacities.getConfiguredMaxResource().getResource()
+                      .equals(Resources.none()))
+                          ? "unlimited"
+                          : capacities.getConfiguredMaxResource().toString())
           .__("Effective Capacity:",
               appendPercent(capacities.getEffectiveMinResource().toString(),
                   capacities.getCapacity() / 100))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db4a4a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 80d16bb..5c4a12e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -511,4 +511,120 @@ public class TestAbsoluteResourceConfiguration {
 
     rm.stop();
   }
+
+  @Test
+  public void testEffectiveResourceAfterIncreasinClusterResource()
+      throws Exception {
+    // create conf with basic queue configuration.
+    CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf);
+
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    @SuppressWarnings("resource")
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    // Add few nodes
+    rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+    rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+    // Get queue object to verify min/max resource configuration.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    ParentQueue qA = (ParentQueue) cs.getQueue(QUEUEA);
+    Assert.assertNotNull(qA);
+    Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB);
+    Assert.assertNotNull(qB);
+    Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+    Assert.assertNotNull(qC);
+    Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+    Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+    Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+    Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+    // add new NM.
+    rm.registerNode("127.0.0.3:1234", 125 * GB, 20);
+
+    // There will be no change in effective resource when nodes are added.
+    // Since configured capacity was based on initial node capacity, a
+    // re configurations is needed to use this added capacity.
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+    rm.stop();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message