spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewo...@apache.org
Subject spark git commit: [HOT FIX] [YARN] Check whether `/lib` exists before listing its files
Date Wed, 03 Dec 2014 21:56:37 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.2 4a71e0853 -> 38cb2c3a3


[HOT FIX] [YARN] Check whether `/lib` exists before listing its files

This is caused by a975dc32799bb8a14f9e1c76defaaa7cfbaf8b53

Author: Andrew Or <andrew@databricks.com>

Closes #3589 from andrewor14/yarn-hot-fix and squashes the following commits:

a4fad5f [Andrew Or] Check whether lib directory exists before listing its files

(cherry picked from commit 90ec643e9af4c8bbb9000edca08c07afb17939c7)
Signed-off-by: Andrew Or <andrew@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/38cb2c3a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/38cb2c3a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/38cb2c3a

Branch: refs/heads/branch-1.2
Commit: 38cb2c3a36a5c9ead4494cbc3dde008c2f0698ce
Parents: 4a71e08
Author: Andrew Or <andrew@databricks.com>
Authored: Wed Dec 3 13:56:23 2014 -0800
Committer: Andrew Or <andrew@databricks.com>
Committed: Wed Dec 3 13:56:35 2014 -0800

----------------------------------------------------------------------
 .../apache/spark/deploy/yarn/ClientBase.scala   | 27 +++++++++++---------
 1 file changed, 15 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/38cb2c3a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 8e4360e..290d994 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -243,18 +243,21 @@ private[spark] trait ClientBase extends Logging {
       val libsURI = new URI(libsDir)
       val jarLinks = ListBuffer.empty[String]
       if (libsURI.getScheme != LOCAL_SCHEME) {
-        val localURI = getQualifiedLocalPath(libsURI).toUri()
-        val jars = FileSystem.get(localURI, hadoopConf).listFiles(new Path(localURI.getPath),
false)
-        while (jars.hasNext) {
-          val jar = jars.next()
-          val name = jar.getPath.getName
-          if (name.startsWith("datanucleus-")) {
-            // copy to remote and add to classpath
-            val src = jar.getPath
-            val destPath = copyFileToRemote(dst, src, replication)
-            distCacheMgr.addResource(fs, hadoopConf, destPath,
-              localResources, LocalResourceType.FILE, name, statCache)
-            jarLinks += name
+        val localPath = getQualifiedLocalPath(libsURI)
+        val localFs = FileSystem.get(localPath.toUri, hadoopConf)
+        if (localFs.exists(localPath)) {
+          val jars = localFs.listFiles(localPath, /* recursive */ false)
+          while (jars.hasNext) {
+            val jar = jars.next()
+            val name = jar.getPath.getName
+            if (name.startsWith("datanucleus-")) {
+              // copy to remote and add to classpath
+              val src = jar.getPath
+              val destPath = copyFileToRemote(dst, src, replication)
+              distCacheMgr.addResource(localFs, hadoopConf, destPath,
+                localResources, LocalResourceType.FILE, name, statCache)
+              jarLinks += name
+            }
           }
         }
       } else {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message