spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewo...@apache.org
Subject git commit: [SPARK-3877][YARN] Throw an exception when application is not successful so that the exit code wil be set to 1 (for branch-1.1)
Date Wed, 22 Oct 2014 22:08:34 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.1 457ef5955 -> eb62094e8


[SPARK-3877][YARN] Throw an exception when application is not successful so that the exit
code wil be set to 1 (for branch-1.1)

This is a patch to fix SPARK-3877 in branch-1.1. See also #2732

Author: zsxwing <zsxwing@gmail.com>

Closes #2748 from zsxwing/SPARK-3877-branch-1.1 and squashes the following commits:

3701984 [zsxwing] Remove System.exit from Client.scala
8681881 [zsxwing] [SPARK-3877] Throw an exception when application is not successful so that
the exit code wil be set to 1


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb62094e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb62094e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb62094e

Branch: refs/heads/branch-1.1
Commit: eb62094e820992355e4cf8ba0e881417f8c6bbfa
Parents: 457ef59
Author: zsxwing <zsxwing@gmail.com>
Authored: Wed Oct 22 15:08:28 2014 -0700
Committer: Andrew Or <andrewor14@gmail.com>
Committed: Wed Oct 22 15:08:28 2014 -0700

----------------------------------------------------------------------
 .../org/apache/spark/deploy/yarn/Client.scala   | 27 ++++++++------------
 .../org/apache/spark/deploy/yarn/Client.scala   | 27 ++++++++------------
 2 files changed, 22 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/eb62094e/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 3607eed..c51ce62 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.ipc.YarnRPC
 import org.apache.hadoop.yarn.util.{Apps, Records}
 
-import org.apache.spark.{Logging, SparkConf}
+import org.apache.spark.{Logging, SparkConf, SparkException}
 
 /**
  * Version of [[org.apache.spark.deploy.yarn.ClientBase]] tailored to YARN's alpha API.
@@ -84,7 +84,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf:
Spa
 
   def run() {
     val appId = runApp()
-    monitorApplication(appId)
+    if (!monitorApplication(appId)) {
+      throw new SparkException("Application is not successful")
+    }
   }
 
   def logClusterResourceDetails() {
@@ -138,10 +140,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration,
spConf: Spa
       )
 
       val state = report.getYarnApplicationState()
-      if (state == YarnApplicationState.FINISHED ||
-        state == YarnApplicationState.FAILED ||
+      if (state == YarnApplicationState.FINISHED) {
+        return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED
+      }
+      if (state == YarnApplicationState.FAILED ||
         state == YarnApplicationState.KILLED) {
-        return true
+        return false
       }
     }
     true
@@ -162,16 +166,7 @@ object Client {
 
     val sparkConf = new SparkConf
 
-    try {
-      val args = new ClientArguments(argStrings, sparkConf)
-      new Client(args, sparkConf).run()
-    } catch {
-      case e: Exception => {
-        Console.err.println(e.getMessage)
-        System.exit(1)
-      }
-    }
-
-    System.exit(0)
+    val args = new ClientArguments(argStrings, sparkConf)
+    new Client(args, sparkConf).run()
   }
 }

http://git-wip-us.apache.org/repos/asf/spark/blob/eb62094e/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 0b55282..da05f7d 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.ipc.YarnRPC
 import org.apache.hadoop.yarn.util.Records
 
-import org.apache.spark.{Logging, SparkConf}
+import org.apache.spark.{Logging, SparkConf, SparkException}
 
 
 /**
@@ -94,7 +94,9 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf:
Spa
 
   def run() {
     val appId = runApp()
-    monitorApplication(appId)
+    if (!monitorApplication(appId)) {
+      throw new SparkException("Application is not successful")
+    }
   }
 
   def logClusterResourceDetails() {
@@ -144,10 +146,12 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration,
spConf: Spa
       )
 
       val state = report.getYarnApplicationState()
-      if (state == YarnApplicationState.FINISHED ||
-        state == YarnApplicationState.FAILED ||
+      if (state == YarnApplicationState.FINISHED) {
+        return report.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED
+      }
+      if (state == YarnApplicationState.FAILED ||
         state == YarnApplicationState.KILLED) {
-        return true
+        return false
       }
     }
     true
@@ -168,17 +172,8 @@ object Client {
     System.setProperty("SPARK_YARN_MODE", "true")
     val sparkConf = new SparkConf()
 
-    try {
-      val args = new ClientArguments(argStrings, sparkConf)
-      new Client(args, sparkConf).run()
-    } catch {
-      case e: Exception => {
-        Console.err.println(e.getMessage)
-        System.exit(1)
-      }
-    }
-
-    System.exit(0)
+    val args = new ClientArguments(argStrings, sparkConf)
+    new Client(args, sparkConf).run()
   }
 
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message