hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vvasu...@apache.org
Subject [29/52] [abbrv] hadoop git commit: YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' has configured child queues (Contributed by Yufei Gu via Daniel Templeton)
Date Mon, 30 Jan 2017 06:04:21 GMT
YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' has configured
child queues
(Contributed by Yufei Gu via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f85b74cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f85b74cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f85b74cc

Branch: refs/heads/YARN-3926
Commit: f85b74ccf9f1c1c1444cc00750b03468cbf40fb9
Parents: 7c1cc30
Author: Daniel Templeton <templedf@apache.org>
Authored: Thu Jan 26 10:31:09 2017 -0800
Committer: Daniel Templeton <templedf@apache.org>
Committed: Thu Jan 26 10:31:09 2017 -0800

----------------------------------------------------------------------
 .../fair/AllocationFileLoaderService.java       | 26 +++---
 .../fair/TestAllocationFileLoaderService.java   | 88 ++++++++++++++++++++
 2 files changed, 101 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85b74cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index cd4a19b..163a265 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -487,6 +487,7 @@ public class AllocationFileLoaderService extends AbstractService {
     Map<ReservationACL, AccessControlList> racls = new HashMap<>();
     NodeList fields = element.getChildNodes();
     boolean isLeaf = true;
+    boolean isReservable = false;
 
     for (int j = 0; j < fields.getLength(); j++) {
       Node fieldNode = fields.item(j);
@@ -558,7 +559,7 @@ public class AllocationFileLoaderService extends AbstractService {
         racls.put(ReservationACL.SUBMIT_RESERVATIONS,
                 new AccessControlList(text));
       } else if ("reservation".equals(field.getTagName())) {
-        isLeaf = false;
+        isReservable = true;
         reservableQueues.add(queueName);
         configuredQueues.get(FSQueueType.PARENT).add(queueName);
       } else if ("allowPreemptionFrom".equals(field.getTagName())) {
@@ -577,22 +578,21 @@ public class AllocationFileLoaderService extends AbstractService {
         isLeaf = false;
       }
     }
-    if (isLeaf) {
-      // if a leaf in the alloc file is marked as type='parent'
-      // then store it under 'parent'
-      if ("parent".equals(element.getAttribute("type"))) {
-        configuredQueues.get(FSQueueType.PARENT).add(queueName);
-      } else {
-        configuredQueues.get(FSQueueType.LEAF).add(queueName);
-      }
+
+    // if a leaf in the alloc file is marked as type='parent'
+    // then store it as a parent queue
+    if (isLeaf && !"parent".equals(element.getAttribute("type"))) {
+      configuredQueues.get(FSQueueType.LEAF).add(queueName);
     } else {
-      if ("parent".equals(element.getAttribute("type"))) {
-        throw new AllocationConfigurationException("Both <reservation> and " +
-            "type=\"parent\" found for queue " + queueName + " which is " +
-            "unsupported");
+      if (isReservable) {
+        throw new AllocationConfigurationException("The configuration settings"
+            + " for " + queueName + " are invalid. A queue element that "
+            + "contains child queue elements or that has the type='parent' "
+            + "attribute cannot also include a reservation element.");
       }
       configuredQueues.get(FSQueueType.PARENT).add(queueName);
     }
+
     // Set default acls if not defined
     // The root queue defaults to all access
     for (QueueACL acl : QueueACL.values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f85b74cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 12c3fa9..c8b9ad8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -618,6 +618,94 @@ public class TestAllocationFileLoaderService {
     allocLoader.reloadAllocations();
   }
 
+  @Test
+  public void testParentTagWithReservation() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"parent\" type=\"parent\">");
+    out.println("<reservation>");
+    out.println("</reservation>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+    allocLoader.init(conf);
+    ReloadListener confHolder = new ReloadListener();
+    allocLoader.setReloadListener(confHolder);
+    try {
+      allocLoader.reloadAllocations();
+    } catch (AllocationConfigurationException ex) {
+      assertEquals(ex.getMessage(), "The configuration settings for root.parent"
+          + " are invalid. A queue element that contains child queue elements"
+          + " or that has the type='parent' attribute cannot also include a"
+          + " reservation element.");
+    }
+  }
+
+  @Test
+  public void testParentWithReservation() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"parent\">");
+    out.println("<reservation>");
+    out.println("</reservation>");
+    out.println(" <queue name=\"child\">");
+    out.println(" </queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+    allocLoader.init(conf);
+    ReloadListener confHolder = new ReloadListener();
+    allocLoader.setReloadListener(confHolder);
+    try {
+      allocLoader.reloadAllocations();
+    } catch (AllocationConfigurationException ex) {
+      assertEquals(ex.getMessage(), "The configuration settings for root.parent"
+          + " are invalid. A queue element that contains child queue elements"
+          + " or that has the type='parent' attribute cannot also include a"
+          + " reservation element.");
+    }
+  }
+
+  @Test
+  public void testParentTagWithChild() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+    out.println("<?xml version=\"1.0\"?>");
+    out.println("<allocations>");
+    out.println("<queue name=\"parent\" type=\"parent\">");
+    out.println(" <queue name=\"child\">");
+    out.println(" </queue>");
+    out.println("</queue>");
+    out.println("</allocations>");
+    out.close();
+
+    AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+    allocLoader.init(conf);
+    ReloadListener confHolder = new ReloadListener();
+    allocLoader.setReloadListener(confHolder);
+    allocLoader.reloadAllocations();
+    AllocationConfiguration queueConf = confHolder.allocConf;
+    // Check whether queue 'parent' and 'child' are loaded successfully
+    assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT)
+        .contains("root.parent"));
+    assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF)
+        .contains("root.parent.child"));
+  }
+
   /**
    * Verify that you can't have the queue name with just a non breaking
    * whitespace in the allocations file.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message