tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject tez git commit: TEZ-3335. DAG client thinks app is still running when app status is null (jeagles)
Date Fri, 07 Oct 2016 21:53:47 GMT
Repository: tez
Updated Branches:
  refs/heads/master dceb365c2 -> 539b8e710


TEZ-3335. DAG client thinks app is still running when app status is null (jeagles)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/539b8e71
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/539b8e71
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/539b8e71

Branch: refs/heads/master
Commit: 539b8e710943b0b79362afc61e246bb83e862ac5
Parents: dceb365
Author: Jonathan Eagles <jeagles@yahoo-inc.com>
Authored: Fri Oct 7 16:53:22 2016 -0500
Committer: Jonathan Eagles <jeagles@yahoo-inc.com>
Committed: Fri Oct 7 16:53:22 2016 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  5 +++-
 .../org/apache/tez/client/TezYarnClient.java    | 12 +++++++++-
 .../org/apache/tez/client/TestTezClient.java    | 25 ++++++++++++++++++++
 3 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/539b8e71/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 63eda94..addd2c0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3335. DAG client thinks app is still running when app status is null
   TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317.
   TEZ-3460. Fix precommit release audit warning.
   TEZ-3368. NPE in DelayedContainerManager
@@ -117,7 +118,8 @@ Release 0.8.5: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
-                
+
+  TEZ-3335. DAG client thinks app is still running when app status is null
   TEZ-3460. Fix precommit release audit warning.
   TEZ-3368. NPE in DelayedContainerManager
   TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map
outputs
@@ -611,6 +613,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3335. DAG client thinks app is still running when app status is null
   TEZ-3460. Fix precommit release audit warning.
   TEZ-3368. NPE in DelayedContainerManager
   TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map
outputs

http://git-wip-us.apache.org/repos/asf/tez/blob/539b8e71/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
index 241e15c..3ac82ac 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;
@@ -86,6 +87,15 @@ public class TezYarnClient extends FrameworkClient {
 
   @Override
   public ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException,
IOException {
-    return yarnClient.getApplicationReport(appId);
+    ApplicationReport report = yarnClient.getApplicationReport(appId);
+    if (report.getYarnApplicationState() == null) {
+      // The state can be null when the ResourceManager does not know about the app but the
YARN
+      // application history server has an incomplete entry for it. Treat this scenario as
if the
+      // application does not exist, since the final app status cannot be determined. This
also
+      // matches the behavior for this scenario if the history server was not configured.
+      throw new ApplicationNotFoundException("YARN reports no state for application "
+          + appId);
+    }
+    return report;
   }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/539b8e71/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 51f36a3..48dfff4 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -48,6 +48,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import com.google.protobuf.ServiceException;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileSystem;
@@ -55,6 +56,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -63,6 +65,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.tez.common.counters.LimitExceededException;
 import org.apache.tez.common.counters.Limits;
@@ -147,6 +151,8 @@ public class TestTezClient {
     ApplicationId appId1 = ApplicationId.newInstance(0, 1);
     YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS);
     when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1);
+    when(yarnClient.getApplicationReport(appId1).getYarnApplicationState()).thenReturn(YarnApplicationState.NEW);
+    when(yarnClient.submitApplication(any(ApplicationSubmissionContext.class))).thenReturn(appId1);
 
     DAGClientAMProtocolBlockingPB sessionAmProxy = mock(DAGClientAMProtocolBlockingPB.class,
RETURNS_DEEP_STUBS);
     when(sessionAmProxy.getAMStatus(any(RpcController.class), any(GetAMStatusRequestProto.class)))
@@ -769,4 +775,23 @@ public class TestTezClient {
     }
     client.stop();
   }
+
+  @Test(timeout = 10000)
+  public void testMissingYarnAppStatus() throws Exception {
+    // verify an app not found exception is thrown when YARN reports a null app status
+    ApplicationId appId1 = ApplicationId.newInstance(0, 1);
+    ApplicationReport mockReport = mock(ApplicationReport.class);
+    when(mockReport.getApplicationId()).thenReturn(appId1);
+    when(mockReport.getYarnApplicationState()).thenReturn(null);
+    YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS);
+    when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1);
+    when(yarnClient.getApplicationReport(appId1)).thenReturn(mockReport);
+    TezYarnClient tezClient = new TezYarnClient(yarnClient);
+    tezClient.init(new TezConfiguration(false), new YarnConfiguration());
+    try {
+      tezClient.getApplicationReport(appId1);
+      fail("getApplicationReport should have thrown");
+    } catch (ApplicationNotFoundException e) {
+    }
+  }
 }


Mime
View raw message