tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject tez git commit: TEZ-3834. TaskSchedulerManager NullPointerException during shutdown when failed to start. Contributed by Jonathan Eagles
Date Tue, 19 Sep 2017 18:20:17 GMT
Repository: tez
Updated Branches:
  refs/heads/master 7236d1563 -> f86128a8d


TEZ-3834. TaskSchedulerManager NullPointerException during shutdown when failed to start.
Contributed by Jonathan Eagles


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/f86128a8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/f86128a8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/f86128a8

Branch: refs/heads/master
Commit: f86128a8dd8a7bbda6b2b61bcaee336b35c6100a
Parents: 7236d15
Author: Jason Lowe <jlowe@apache.org>
Authored: Tue Sep 19 13:17:11 2017 -0500
Committer: Jason Lowe <jlowe@apache.org>
Committed: Tue Sep 19 13:17:11 2017 -0500

----------------------------------------------------------------------
 .../tez/dag/app/rm/TaskSchedulerManager.java    | 24 ++++++++++++++------
 .../dag/app/rm/TestTaskSchedulerManager.java    | 17 ++++++++++++++
 2 files changed, 34 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/f86128a8/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
index 640e8f6..7c1b926 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java
@@ -684,12 +684,14 @@ public class TaskSchedulerManager extends AbstractService implements
 
   public void initiateStop() {
     for (int i = 0 ; i < taskSchedulers.length ; i++) {
-      try {
-        taskSchedulers[i].getTaskScheduler().initiateStop();
-      } catch (Exception e) {
-        // Ignore for now as scheduler stop invoked on shutdown
-        LOG.error("Failed to do a clean initiateStop for Scheduler: "
-            + Utils.getTaskSchedulerIdentifierString(i, appContext), e);
+      if (taskSchedulers[i] != null) {
+        try {
+          taskSchedulers[i].getTaskScheduler().initiateStop();
+        } catch (Exception e) {
+          // Ignore for now as scheduler stop invoked on shutdown
+          LOG.error("Failed to do a clean initiateStop for Scheduler: "
+              + Utils.getTaskSchedulerIdentifierString(i, appContext), e);
+        }
       }
     }
   }
@@ -978,13 +980,21 @@ public class TaskSchedulerManager extends AbstractService implements
   }
 
   public boolean hasUnregistered() {
+    // Only return true if all task schedulers that were registered successfully unregister
+    if (taskSchedulers.length == 0) {
+      return false;
+    }
     boolean result = true;
-    for (int i = 0 ; i < taskSchedulers.length ; i++) {
+    for (int i = 0; i < taskSchedulers.length; i++) {
       // Explicitly not catching any exceptions around this API
       // No clear route to recover. Better to crash.
+      if (taskSchedulers[i] == null) {
+        return false;
+      }
       try {
         result = result & this.taskSchedulers[i].hasUnregistered();
       } catch (Exception e) {
+        result = false;
         String msg = "Error in TaskScheduler when checking if a scheduler has unregistered"
             + ", scheduler=" + Utils.getTaskSchedulerIdentifierString(i, appContext);
         LOG.error(msg, e);

http://git-wip-us.apache.org/repos/asf/tez/blob/f86128a8/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
index 43805f3..5df25de 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java
@@ -570,6 +570,23 @@ public class TestTaskSchedulerManager {
 
   @SuppressWarnings("unchecked")
   @Test(timeout = 5000)
+  public void testShutdownBeforeStartTaskScheduler() {
+    Configuration conf = new TezConfiguration();
+    AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
+    doReturn(conf).when(appContext).getAMConf();
+
+    List<NamedEntityDescriptor> list = new LinkedList<>();
+    list.add(null);
+
+    TaskSchedulerManager taskSchedulerManager =
+        new TaskSchedulerManager(appContext, null, null,
+            null, null, list, false,null);
+    assertFalse("Should not return true unless actually unregistered successfully",
+        taskSchedulerManager.hasUnregistered());
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test(timeout = 5000)
   public void testReportFailureFromTaskScheduler() {
     String dagName = DAG_NAME;
     Configuration conf = new TezConfiguration();


Mime
View raw message