tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-900. Confusing message for incorrect queue for some tez examples. (Chang Li via hitesh)
Date Fri, 06 Feb 2015 22:52:28 GMT
Repository: tez
Updated Branches:
  refs/heads/master 2b12f4cba -> 2b5eeea52


TEZ-900. Confusing message for incorrect queue for some tez examples. (Chang Li via hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2b5eeea5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2b5eeea5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2b5eeea5

Branch: refs/heads/master
Commit: 2b5eeea52b63ee10aa92ce1d7cd7ac88aff6a730
Parents: 2b12f4c
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Feb 6 14:35:38 2015 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Feb 6 14:35:38 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/tez/client/TezYarnClient.java    |  8 ++++-
 .../org/apache/tez/client/TestTezClient.java    |  8 +++++
 .../java/org/apache/tez/test/TestTezJobs.java   | 35 ++++++++++++++++++++
 4 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ab46d64..584f49a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -180,6 +180,7 @@ TEZ-UI CHANGES (TEZ-8):
 Release 0.5.4: Unreleased
 
 ALL CHANGES:
+  TEZ-900. Confusing message for incorrect queue for some tez examples.
   TEZ-2036. OneToOneEdgeManager should enforce that source and destination
   tasks have same number
   TEZ-1895. Vertex reRunning should decrease successfulMembers of VertexGroupInfo.

http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
index d58a0bb..eae1909 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -66,7 +67,12 @@ public class TezYarnClient extends FrameworkClient {
 
   @Override
   public ApplicationId submitApplication(ApplicationSubmissionContext appSubmissionContext)
throws YarnException, IOException {
-    return yarnClient.submitApplication(appSubmissionContext);
+	ApplicationId appId= yarnClient.submitApplication(appSubmissionContext);
+    ApplicationReport appReport = getApplicationReport(appId);
+    if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED){
+      throw new IOException("Failed to submit application to YARN: " + appReport.getDiagnostics());
+    }
+    return appId;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index f4d4112..14d4d7f 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -141,6 +141,8 @@ public class TestTezClient {
     TezClientForTest client = configure(lrs, isSession);
     
     ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
+    when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+    .thenReturn(YarnApplicationState.RUNNING);
     client.start();
     verify(client.mockYarnClient, times(1)).init((Configuration)any());
     verify(client.mockYarnClient, times(1)).start();
@@ -197,6 +199,8 @@ public class TestTezClient {
     when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId())
         .thenReturn(appId2);
     
+    when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState())
+    .thenReturn(YarnApplicationState.RUNNING);
     dag = DAG.create("DAG").addVertex(
         Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1,
1)));
     dagClient = client.submitDAG(dag);
@@ -269,6 +273,8 @@ public class TestTezClient {
   
   public void testMultipleSubmissionsJob(boolean isSession) throws Exception {
     TezClientForTest client1 = configure(new HashMap<String, LocalResource>(), isSession);
+    when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState())
+    .thenReturn(YarnApplicationState.RUNNING);
     client1.start();
     
     String mockLR1Name = "LR1";
@@ -287,6 +293,8 @@ public class TestTezClient {
     client1.submitDAG(dag);
     
     TezClientForTest client2 = configure();
+    when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState())
+    .thenReturn(YarnApplicationState.RUNNING);
     client2.start();
     
     // verify resubmission of same dag to new client (simulates submission error resulting
in the

http://git-wip-us.apache.org/repos/asf/tez/blob/2b5eeea5/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index adf27e8..b9cb491 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -672,6 +672,41 @@ public class TestTezJobs {
 
   }
 
+  @Test(timeout = 60000)
+  public void testInvalidQueueSubmission() throws Exception {
+
+    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
+    YarnClient yarnClient = YarnClient.createYarnClient();
+    try {
+
+      yarnClient.init(mrrTezCluster.getConfig());
+      yarnClient.start();
+
+      SimpleSessionExample job = new SimpleSessionExample();
+      tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
+      tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");
+
+      String[] inputPaths = new String[1];
+      String[] outputPaths = new String[1];
+      String inputDirStr = "/tmp/owc-input";
+      inputPaths[0] = inputDirStr;
+      Path inputDir = new Path(inputDirStr);
+      remoteFs.mkdirs(inputDir);
+      String outputDirStr = "/tmp/owc-output";
+      outputPaths[0] = outputDirStr;
+      job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
+          StringUtils.join(",", outputPaths), "2" }, null);
+      fail("Job submission should have thrown an exception");
+    } catch(IOException e) {
+      Assert.assertTrue(e.getMessage().contains("Failed to submit application to YARN"));
+    } finally {
+      if (yarnClient != null) {
+        yarnClient.stop();
+      }
+    }
+
+  }
+
   @Test (timeout=60000)
   public void testVertexOrder() throws Exception {
     TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());


Mime
View raw message