spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pwend...@apache.org
Subject spark git commit: [SPARK-2703][Core]Make Tachyon related unit tests execute without deploying a Tachyon system locally.
Date Mon, 10 Nov 2014 07:48:30 GMT
Repository: spark
Updated Branches:
  refs/heads/master 227488d8c -> bd86cb173


[SPARK-2703][Core]Make Tachyon related unit tests execute without deploying a Tachyon system
locally.

Make Tachyon related unit tests execute without deploying a Tachyon system locally.

Author: RongGu <gurongwalker@gmail.com>

Closes #3030 from RongGu/SPARK-2703 and squashes the following commits:

ad08827 [RongGu] Make Tachyon related unit tests execute without deploying a Tachyon system
locally


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bd86cb17
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bd86cb17
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bd86cb17

Branch: refs/heads/master
Commit: bd86cb1738800a0aa4c88b9afdba2f97ac6cbf25
Parents: 227488d
Author: RongGu <gurongwalker@gmail.com>
Authored: Sun Nov 9 23:48:15 2014 -0800
Committer: Patrick Wendell <pwendell@gmail.com>
Committed: Sun Nov 9 23:48:15 2014 -0800

----------------------------------------------------------------------
 core/pom.xml                                             |  7 +++++++
 .../org/apache/spark/storage/BlockManagerSuite.scala     | 11 +++++++++--
 project/SparkBuild.scala                                 |  2 ++
 3 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/bd86cb17/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 41296e0..92e9f1f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -206,6 +206,13 @@
     </dependency>
     <dependency>
       <groupId>org.tachyonproject</groupId>
+      <artifactId>tachyon</artifactId>
+      <version>0.5.0</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.tachyonproject</groupId>
       <artifactId>tachyon-client</artifactId>
       <version>0.5.0</version>
       <exclusions>

http://git-wip-us.apache.org/repos/asf/spark/blob/bd86cb17/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 9529502..86503c9 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -36,6 +36,7 @@ import org.mockito.Mockito.{mock, when}
 import org.scalatest.{BeforeAndAfter, FunSuite, Matchers, PrivateMethodTester}
 import org.scalatest.concurrent.Eventually._
 import org.scalatest.concurrent.Timeouts._
+import tachyon.master.LocalTachyonCluster
 
 import org.apache.spark.{MapOutputTrackerMaster, SparkConf, SparkContext, SecurityManager}
 import org.apache.spark.executor.DataReadMethod
@@ -536,9 +537,14 @@ class BlockManagerSuite extends FunSuite with Matchers with BeforeAndAfter
   }
 
   test("tachyon storage") {
-    // TODO Make the spark.test.tachyon.enable true after using tachyon 0.5.0 testing jar.
-    val tachyonUnitTestEnabled = conf.getBoolean("spark.test.tachyon.enable", false)
+    val tachyonUnitTestEnabled = conf.getBoolean("spark.test.tachyon.enable", true)
     if (tachyonUnitTestEnabled) {
+      val tachyonCluster = new LocalTachyonCluster(30000000)
+      tachyonCluster.start()
+      val tachyonURL = tachyon.Constants.HEADER +
+        tachyonCluster.getMasterHostname() + ":" + tachyonCluster.getMasterPort()
+      conf.set("spark.tachyonStore.url", tachyonURL)
+      conf.set("spark.tachyonStore.folderName", "app-test")
       store = makeBlockManager(1200)
       val a1 = new Array[Byte](400)
       val a2 = new Array[Byte](400)
@@ -549,6 +555,7 @@ class BlockManagerSuite extends FunSuite with Matchers with BeforeAndAfter
       assert(store.getSingle("a3").isDefined, "a3 was in store")
       assert(store.getSingle("a2").isDefined, "a2 was in store")
       assert(store.getSingle("a1").isDefined, "a1 was in store")
+      tachyonCluster.stop()
     } else {
       info("tachyon storage test disabled.")
     }

http://git-wip-us.apache.org/repos/asf/spark/blob/bd86cb17/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 657e4b4..351e57a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -360,6 +360,8 @@ object TestSettings {
     testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
     // Enable Junit testing.
     libraryDependencies += "com.novocode" % "junit-interface" % "0.9" % "test",
+    // Enable Tachyon local testing.
+    libraryDependencies += "org.tachyonproject" % "tachyon" % "0.5.0" % "test" classifier
"tests",
     // Only allow one test at a time, even across projects, since they run in the same JVM
     parallelExecution in Test := false,
     concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message