spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pwend...@apache.org
Subject [03/20] spark git commit: Setting test jars on executor classpath during tests from sbt.
Date Fri, 07 Nov 2014 19:31:25 GMT
Setting test jars on executor classpath during tests from sbt.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4bcf66f2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4bcf66f2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4bcf66f2

Branch: refs/heads/scala-2.11-prashant
Commit: 4bcf66f2c9421f0c50e5a4578a80812aea4b8b10
Parents: 29cc4d9
Author: Prashant Sharma <prashant.s@imaginea.com>
Authored: Wed Nov 5 16:51:09 2014 +0530
Committer: Prashant Sharma <prashant.s@imaginea.com>
Committed: Fri Nov 7 11:22:47 2014 +0530

----------------------------------------------------------------------
 bin/compute-classpath.sh | 6 ------
 project/SparkBuild.scala | 8 +++++---
 2 files changed, 5 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/4bcf66f2/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 108c9af..14e972f 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -137,9 +137,6 @@ if [ -n "$datanucleus_jars" ]; then
   fi
 fi
 
-test_jars=$(find "$FWDIR"/lib_managed/test \( -name '*jar' -a -type f \) 2>/dev/null |
\
-    tr "\n" : | sed s/:$//g)
-
 # Add test classes if we're running from SBT or Maven with SPARK_TESTING set to 1
 if [[ $SPARK_TESTING == 1 ]]; then
   CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/test-classes"
@@ -151,9 +148,6 @@ if [[ $SPARK_TESTING == 1 ]]; then
   CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SPARK_SCALA_VERSION/test-classes"
   CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SPARK_SCALA_VERSION/test-classes"
   CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/test-classes"
-  if [[ $SPARK_SCALA_VERSION == "2.11" ]]; then
-      CLASSPATH="$CLASSPATH:$test_jars"
-  fi
 fi
 
 # Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail !

http://git-wip-us.apache.org/repos/asf/spark/blob/4bcf66f2/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 0d8adcb..9e46a11 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -31,8 +31,8 @@ object BuildCommons {
   private val buildLocation = file(".").getAbsoluteFile.getParentFile
 
   val allProjects@Seq(bagel, catalyst, core, graphx, hive, hiveThriftServer, mllib, repl,
-  sql, networkCommon, networkShuffle, streaming, streamingFlumeSink, streamingFlume, streamingKafka,
-  streamingMqtt, streamingTwitter, streamingZeromq) =
+    sql, networkCommon, networkShuffle, streaming, streamingFlumeSink, streamingFlume, streamingKafka,
+    streamingMqtt, streamingTwitter, streamingZeromq) =
     Seq("bagel", "catalyst", "core", "graphx", "hive", "hive-thriftserver", "mllib", "repl",
       "sql", "network-common", "network-shuffle", "streaming", "streaming-flume-sink",
       "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter",
@@ -361,8 +361,10 @@ object TestSettings {
       .map { case (k,v) => s"-D$k=$v" }.toSeq,
     javaOptions in Test ++= "-Xmx3g -XX:PermSize=128M -XX:MaxNewSize=256m -XX:MaxPermSize=1g"
       .split(" ").toSeq,
+    javaOptions in Test += 
+      "-Dspark.executor.extraClassPath=" + (fullClasspath in Test).value.files.
+      map(_.getAbsolutePath).mkString(":").stripSuffix(":"),
     javaOptions += "-Xmx3g",
-    retrievePattern := "[conf]/[artifact](-[revision]).[ext]",
     // Show full stack trace and duration in test cases.
     testOptions in Test += Tests.Argument("-oDF"),
     testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message