hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rohithsharm...@apache.org
Subject hadoop git commit: YARN-7289. Application lifetime does not work with FairScheduler. Contributed by Miklos Szegedi.
Date Fri, 27 Oct 2017 17:48:08 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 e084d4355 -> d9b636f2f


YARN-7289. Application lifetime does not work with FairScheduler. Contributed by Miklos Szegedi.

(cherry picked from commit 5c799ecf09617f634c41bba26d341b2ad45ee852)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9b636f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9b636f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9b636f2

Branch: refs/heads/branch-3.0
Commit: d9b636f2f47b2c5f4087288e85d3f75c9236b29c
Parents: e084d43
Author: Rohith Sharma K S <rohithsharmaks@apache.org>
Authored: Fri Oct 27 22:46:38 2017 +0530
Committer: Rohith Sharma K S <rohithsharmaks@apache.org>
Committed: Fri Oct 27 23:14:55 2017 +0530

----------------------------------------------------------------------
 .../scheduler/fair/FairScheduler.java           |  6 ++
 .../rmapp/TestApplicationLifetimeMonitor.java   | 61 +++++++++++++++-----
 2 files changed, 51 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9b636f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 81f6f9d..cbfa9f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1802,4 +1802,10 @@ public class FairScheduler extends
   ReadLock getSchedulerReadLock() {
     return this.readLock;
   }
+
+  @Override
+  public long checkAndGetApplicationLifetime(String queueName, long lifetime) {
+    // Lifetime is the application lifetime by default.
+    return lifetime;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9b636f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
index a7808d7..4f88480 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
@@ -22,7 +22,9 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -54,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
 import org.apache.hadoop.yarn.util.Times;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
@@ -61,19 +64,36 @@ import org.apache.log4j.Logger;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /**
  * Test class for application life time monitor feature test.
  */
+@RunWith(Parameterized.class)
 public class TestApplicationLifetimeMonitor {
   private YarnConfiguration conf;
 
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    Collection<Object[]> params = new ArrayList<Object[]>();
+    params.add(new Object[]{CapacityScheduler.class});
+    params.add(new Object[]{FairScheduler.class});
+    return params;
+  }
+
+  private Class scheduler;
+
+  public TestApplicationLifetimeMonitor(Class schedulerParameter) {
+    scheduler = schedulerParameter;
+  }
+
   @Before
   public void setup() throws IOException {
     conf = new YarnConfiguration();
     // Always run for CS, since other scheduler do not support this.
     conf.setClass(YarnConfiguration.RM_SCHEDULER,
-        CapacityScheduler.class, ResourceScheduler.class);
+        scheduler, ResourceScheduler.class);
     Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
     UserGroupInformation.setConfiguration(conf);
@@ -82,15 +102,20 @@ public class TestApplicationLifetimeMonitor {
   }
 
   @Test(timeout = 60000)
-  public void testApplicationLifetimeMonitor() throws Exception {
+  public void testApplicationLifetimeMonitor()
+      throws Exception {
     MockRM rm = null;
     try {
       long maxLifetime = 30L;
       long defaultLifetime = 15L;
 
-      YarnConfiguration newConf =
-          new YarnConfiguration(setUpCSQueue(maxLifetime, defaultLifetime));
-      conf = new YarnConfiguration(newConf);
+      YarnConfiguration newConf;
+      if (scheduler.equals(CapacityScheduler.class)) {
+        // Since there is limited lifetime monitoring support in fair scheduler
+        // it does not need queue setup
+        conf =
+            new YarnConfiguration(setUpCSQueue(maxLifetime, defaultLifetime));
+      }
       rm = new MockRM(conf);
       rm.start();
 
@@ -172,17 +197,21 @@ public class TestApplicationLifetimeMonitor {
       Assert.assertTrue("Application killed before lifetime value",
           app2.getFinishTime() > afterUpdate);
 
-      rm.waitForState(app3.getApplicationId(), RMAppState.KILLED);
-
-      // app4 submitted exceeding queue max lifetime, so killed after queue max
-      // lifetime.
-      rm.waitForState(app4.getApplicationId(), RMAppState.KILLED);
-      long totalTimeRun = (app4.getFinishTime() - app4.getSubmitTime()) / 1000;
-      Assert.assertTrue("Application killed before lifetime value",
-          totalTimeRun > maxLifetime);
-      Assert.assertTrue(
-          "Application killed before lifetime value " + totalTimeRun,
-          totalTimeRun < maxLifetime + 10L);
+      if (scheduler.equals(CapacityScheduler.class)) {
+        // Supported only on capacity scheduler
+        rm.waitForState(app3.getApplicationId(), RMAppState.KILLED);
+
+        // app4 submitted exceeding queue max lifetime,
+        // so killed after queue max lifetime.
+        rm.waitForState(app4.getApplicationId(), RMAppState.KILLED);
+        long totalTimeRun =
+            (app4.getFinishTime() - app4.getSubmitTime()) / 1000;
+        Assert.assertTrue("Application killed before lifetime value",
+            totalTimeRun > maxLifetime);
+        Assert.assertTrue(
+            "Application killed before lifetime value " + totalTimeRun,
+            totalTimeRun < maxLifetime + 10L);
+      }
     } finally {
       stopRM(rm);
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message