spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pwend...@apache.org
Subject git commit: SPARK-1652: Spark submit should fail gracefully if YARN not enabled
Date Tue, 29 Apr 2014 00:27:06 GMT
Repository: spark
Updated Branches:
  refs/heads/master 8421034e7 -> cae054aaf


SPARK-1652: Spark submit should fail gracefully if YARN not enabled

Author: Patrick Wendell <pwendell@gmail.com>

Closes #579 from pwendell/spark-submit-yarn-2 and squashes the following commits:

05e1b11 [Patrick Wendell] Small fix
d2a40ad [Patrick Wendell] SPARK-1652: Spark submit should fail gracefully if YARN support
not enabled


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cae054aa
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cae054aa
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cae054aa

Branch: refs/heads/master
Commit: cae054aaf41ca0ee585231896db67169b61af689
Parents: 8421034
Author: Patrick Wendell <pwendell@gmail.com>
Authored: Mon Apr 28 17:26:57 2014 -0700
Committer: Patrick Wendell <pwendell@gmail.com>
Committed: Mon Apr 28 17:26:57 2014 -0700

----------------------------------------------------------------------
 .../main/scala/org/apache/spark/deploy/SparkSubmit.scala  | 10 ++++++++++
 core/src/main/scala/org/apache/spark/util/Utils.scala     |  6 ++++++
 2 files changed, 16 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/cae054aa/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 24edc60..c463ee0 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -23,6 +23,7 @@ import java.net.{URI, URL}
 import scala.collection.mutable.{ArrayBuffer, HashMap, Map}
 
 import org.apache.spark.executor.ExecutorURLClassLoader
+import org.apache.spark.util.Utils
 
 /**
  * Scala code behind the spark-submit script.  The script handles setting up the classpath
with
@@ -128,6 +129,15 @@ object SparkSubmit {
       childArgs += ("--class", appArgs.mainClass)
     }
 
+    if (clusterManager == YARN) {
+      // The choice of class is arbitrary, could use any spark-yarn class
+      if (!Utils.classIsLoadable("org.apache.spark.deploy.yarn.Client") && !Utils.isTesting)
{
+        val msg = "Could not load YARN classes. This copy of Spark may not have been compiled
" +
+          "with YARN support."
+        throw new Exception(msg)
+      }
+    }
+
     val options = List[OptionAssigner](
       new OptionAssigner(appArgs.master, ALL_CLUSTER_MGRS, false, sysProp = "spark.master"),
       new OptionAssigner(appArgs.driverExtraClassPath, STANDALONE | YARN, true,

http://git-wip-us.apache.org/repos/asf/spark/blob/cae054aa/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index b678604..79f314c 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -28,6 +28,7 @@ import scala.collection.Map
 import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
 import scala.reflect.ClassTag
+import scala.util.Try
 
 import com.google.common.io.Files
 import org.apache.commons.lang.SystemUtils
@@ -137,6 +138,11 @@ private[spark] object Utils extends Logging {
   def getContextOrSparkClassLoader =
     Option(Thread.currentThread().getContextClassLoader).getOrElse(getSparkClassLoader)
 
+  /** Determines whether the provided class is loadable in the current thread. */
+  def classIsLoadable(clazz: String): Boolean = {
+    Try { Class.forName(clazz, false, getContextOrSparkClassLoader) }.isSuccess
+  }
+
   /**
    * Primitive often used when writing {@link java.nio.ByteBuffer} to {@link java.io.DataOutput}.
    */


Mime
View raw message