hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vinayakum...@apache.org
Subject [23/29] hadoop git commit: YARN-2869. CapacityScheduler should trim sub queue names when parse configuration. Contributed by Wangda Tan
Date Mon, 08 Dec 2014 09:15:46 GMT
YARN-2869. CapacityScheduler should trim sub queue names when parse configuration. Contributed
by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e69af836
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e69af836
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e69af836

Branch: refs/heads/HDFS-EC
Commit: e69af836f34f16fba565ab112c9bf0d367675b16
Parents: 475c6b4
Author: Jian He <jianhe@apache.org>
Authored: Fri Dec 5 17:33:39 2014 -0800
Committer: Jian He <jianhe@apache.org>
Committed: Fri Dec 5 17:33:39 2014 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |   3 +
 .../CapacitySchedulerConfiguration.java         |  10 +-
 .../scheduler/capacity/TestQueueParsing.java    | 110 +++++++++++++++++++
 3 files changed, 122 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0b88959..0d7a843 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -194,6 +194,9 @@ Release 2.7.0 - UNRELEASED
     YARN-2461. Fix PROCFS_USE_SMAPS_BASED_RSS_ENABLED property in
     YarnConfiguration. (rchiang via rkanter)
 
+    YARN-2869. CapacityScheduler should trim sub queue names when parse
+    configuration. (Wangda Tan via jianhe)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 0a49224..5bbb436 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -260,7 +260,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     }
   }
 
-  private String getQueuePrefix(String queue) {
+  static String getQueuePrefix(String queue) {
     String queueName = PREFIX + queue + DOT;
     return queueName;
   }
@@ -538,6 +538,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   public String[] getQueues(String queue) {
     LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
     String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
+    List<String> trimmedQueueNames = new ArrayList<String>();
+    if (null != queues) {
+      for (String s : queues) {
+        trimmedQueueNames.add(s.trim());
+      }
+      queues = trimmedQueueNames.toArray(new String[0]);
+    }
+ 
     LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) + 
         ", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues)));
     return queues;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index cf2e5ce..5a9fbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -82,6 +82,56 @@ public class TestQueueParsing {
     ServiceOperations.stopQuietly(capacityScheduler);
   }
   
+  private void setupQueueConfigurationWithSpacesShouldBeTrimmed(
+      CapacitySchedulerConfiguration conf) {
+    // Define top-level queues
+    conf.set(
+        CapacitySchedulerConfiguration
+            .getQueuePrefix(CapacitySchedulerConfiguration.ROOT)
+            + CapacitySchedulerConfiguration.QUEUES, " a ,b, c");
+
+    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+    conf.setCapacity(A, 10);
+    conf.setMaximumCapacity(A, 15);
+    
+    final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+    conf.setCapacity(B, 20);
+    
+    final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+    conf.setCapacity(C, 70);
+    conf.setMaximumCapacity(C, 70);
+  }
+  
+  private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed(
+      CapacitySchedulerConfiguration conf) {
+    // Define top-level queues
+    conf.set(
+        CapacitySchedulerConfiguration
+            .getQueuePrefix(CapacitySchedulerConfiguration.ROOT)
+            + CapacitySchedulerConfiguration.QUEUES, " a ,b, c");
+
+    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+    conf.setCapacity(A, 10);
+    conf.setMaximumCapacity(A, 15);
+
+    final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+    conf.setCapacity(B, 20);
+
+    final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+    conf.setCapacity(C, 70);
+    conf.setMaximumCapacity(C, 70);
+
+    // sub queues for A
+    conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A)
+        + CapacitySchedulerConfiguration.QUEUES, "a1, a2 ");
+
+    final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1";
+    conf.setCapacity(A1, 60);
+
+    final String A2 = CapacitySchedulerConfiguration.ROOT + ".a.a2";
+    conf.setCapacity(A2, 40);
+  }
+  
   private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
     
     // Define top-level queues
@@ -659,4 +709,64 @@ public class TestQueueParsing {
         DELTA);
     capacityScheduler.stop();
   }
+  
+  @Test
+  public void testQueueParsingShouldTrimSpaces() throws Exception {
+    CapacitySchedulerConfiguration csConf = 
+      new CapacitySchedulerConfiguration();
+    setupQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
+    YarnConfiguration conf = new YarnConfiguration(csConf);
+
+    CapacityScheduler capacityScheduler = new CapacityScheduler();
+    capacityScheduler.setConf(conf);
+    capacityScheduler.setRMContext(TestUtils.getMockRMContext());
+    capacityScheduler.init(conf);
+    capacityScheduler.start();
+    capacityScheduler.reinitialize(conf, TestUtils.getMockRMContext());
+    
+    CSQueue a = capacityScheduler.getQueue("a");
+    Assert.assertNotNull(a);
+    Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA);
+    
+    CSQueue c = capacityScheduler.getQueue("c");
+    Assert.assertNotNull(c);
+    Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA);
+  }
+  
+  @Test
+  public void testNestedQueueParsingShouldTrimSpaces() throws Exception {
+    CapacitySchedulerConfiguration csConf = 
+      new CapacitySchedulerConfiguration();
+    setupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
+    YarnConfiguration conf = new YarnConfiguration(csConf);
+
+    CapacityScheduler capacityScheduler = new CapacityScheduler();
+    capacityScheduler.setConf(conf);
+    capacityScheduler.setRMContext(TestUtils.getMockRMContext());
+    capacityScheduler.init(conf);
+    capacityScheduler.start();
+    capacityScheduler.reinitialize(conf, TestUtils.getMockRMContext());
+    
+    CSQueue a = capacityScheduler.getQueue("a");
+    Assert.assertNotNull(a);
+    Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA);
+    
+    CSQueue c = capacityScheduler.getQueue("c");
+    Assert.assertNotNull(c);
+    Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA);
+    
+    CSQueue a1 = capacityScheduler.getQueue("a1");
+    Assert.assertNotNull(a1);
+    Assert.assertEquals(0.10 * 0.6, a1.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.15, a1.getAbsoluteMaximumCapacity(), DELTA);
+    
+    CSQueue a2 = capacityScheduler.getQueue("a2");
+    Assert.assertNotNull(a2);
+    Assert.assertEquals(0.10 * 0.4, a2.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.15, a2.getAbsoluteMaximumCapacity(), DELTA);
+  }
 }


Mime
View raw message