tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3368. NPE in DelayedContainerManager (Jason Lowe via jeagles)
Date Fri, 07 Oct 2016 17:16:25 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.8 7adfb6dc7 -> 9a6988512


TEZ-3368. NPE in DelayedContainerManager (Jason Lowe via jeagles)

(cherry picked from commit 5bd5613fca24bb521f0d8a9dd2cace4f80c5e90a)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9a698851
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9a698851
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9a698851

Branch: refs/heads/branch-0.8
Commit: 9a69885124f1802f2acd5cbd5c652dbca7a6c3a5
Parents: 7adfb6d
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Fri Oct 7 11:32:17 2016 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Fri Oct 7 12:16:01 2016 -0500

----------------------------------------------------------------------
 CHANGES.txt                                        |  2 ++
 .../apache/tez/dag/app/rm/TezAMRMClientAsync.java  | 17 +++++++----------
 .../tez/dag/app/rm/YarnTaskSchedulerService.java   |  9 +++++++++
 3 files changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/9a698851/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 390c528..e27b71a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -8,6 +8,7 @@ INCOMPATIBLE CHANGES
 ALL CHANGES:
 
   TEZ-3460. Fix precommit release audit warning
+  TEZ-3368. NPE in DelayedContainerManager
   TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map
outputs
   TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly.
   TEZ-3000. Fix TestContainerReuse.
@@ -511,6 +512,7 @@ INCOMPATIBLE CHANGES
 ALL CHANGES:
 
   TEZ-3460. Fix precommit release audit warning
+  TEZ-3368. NPE in DelayedContainerManager
   TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map
outputs
   TEZ-3426. Second AM attempt launched for session mode and recovery disabled for certain
cases
   TEZ-3009. Errors that occur during container task acquisition are not logged.

http://git-wip-us.apache.org/repos/asf/tez/blob/9a698851/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
index 7b3df84..40bc3aa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.app.rm;
 
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -74,12 +74,10 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends
AMRMClientAs
   }
   
   public synchronized Priority getTopPriority() {
-    Iterator<Priority> iter =
-        knownRequestsByPriority.descendingKeySet().iterator();
-    if (!iter.hasNext()) {
+    if (knownRequestsByPriority.isEmpty()) {
       return null;
     }
-    return iter.next();
+    return knownRequestsByPriority.lastKey();
   }
   
   // Remove after YARN-1723 is fixed
@@ -132,13 +130,12 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends
AMRMClientAs
         String resourceName, Resource capability) {
     // Sort based on reverse order. By default, Priority ordering is based on
     // highest numeric value being considered to be lowest priority.
-    Iterator<Priority> iter =
-      knownRequestsByPriority.descendingKeySet().iterator();
-    if (!iter.hasNext()) {
+    Map.Entry<Priority,LocalityRequestCounter> entry = knownRequestsByPriority.lastEntry();
+    if (entry == null || entry.getValue() == null) {
       return Collections.emptyList();
     }
-    Priority p = iter.next();
-    LocalityRequestCounter lrc = knownRequestsByPriority.get(p);
+    Priority p = entry.getKey();
+    LocalityRequestCounter lrc = entry.getValue();
     if (lrc.localityRequests.get() == 0) {
       // Fallback to ANY if there are no pending requests that require
       // locality matching

http://git-wip-us.apache.org/repos/asf/tez/blob/9a698851/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index 6dfc7b9..5087d0d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -47,6 +47,7 @@ import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.commons.math3.random.RandomDataGenerator;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -1958,6 +1959,14 @@ public class YarnTaskSchedulerService extends TaskScheduler
     
     @Override
     public void run() {
+      try {
+        mainLoop();
+      } catch (Throwable e) {
+        ExitUtil.terminate(1, e);
+      }
+    }
+
+    private void mainLoop() {
       while(running) {
         // Try assigning all containers if there's a request to do so.
         if (tryAssigningAll) {


Mime
View raw message