mnemonic-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject incubator-mnemonic git commit: MNEMONIC-246: Make test data generated from Random for unit tests
Date Tue, 25 Apr 2017 21:54:35 GMT
Repository: incubator-mnemonic
Updated Branches:
  refs/heads/master 1e6bf9558 -> 3e628a6bb


MNEMONIC-246: Make test data generated from Random for unit tests


Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/3e628a6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/3e628a6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/3e628a6b

Branch: refs/heads/master
Commit: 3e628a6bbef915c17cf13458e965f5c699d2b767
Parents: 1e6bf95
Author: Wang, Gang(Gary) <gang1.wang@intel.com>
Authored: Tue Apr 25 14:53:50 2017 -0700
Committer: Wang, Gang(Gary) <gang1.wang@intel.com>
Committed: Tue Apr 25 14:53:50 2017 -0700

----------------------------------------------------------------------
 .../org/apache/mnemonic/sessions/DurableOutputSession.java  | 2 +-
 .../org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala      | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3e628a6b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
----------------------------------------------------------------------
diff --git a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
index e21b5a8..fad491f 100644
--- a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
+++ b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
@@ -50,7 +50,7 @@ public abstract class DurableOutputSession<V, A extends RestorableAllocator<A>>
 
   /**
    * Initialize the next pool, must be called before use
-   *
+   * and after configuration
    * @return true if success
    */
   protected abstract boolean initNextPool();

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3e628a6b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
index 6349ce5..ac33e7c 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
@@ -17,6 +17,7 @@
 
 package org.apache.mnemonic.spark.rdd;
 
+import scala.util._
 import org.apache.mnemonic.spark.TestSpec
 import org.apache.spark.SparkConf
 import org.apache.spark.SparkContext
@@ -33,7 +34,8 @@ class DurableRDDSpec extends TestSpec {
   val defaultSlotKeyId = 2L
   val defaultPartitionSize = 1024 * 1024 * 1024L
   val defaultBaseDirectory = "."
-  val defaultNumOfRecordsPerPartition = 5000L
+  val defaultNumOfPartitions = 8
+  val defaultNumOfRecordsPerPartition = 5000
 
   behavior of "A DurableRDD"
 
@@ -43,8 +45,9 @@ class DurableRDDSpec extends TestSpec {
         .setAppName("Test")
     val sc = new SparkContext(conf)
     // sc.getConf.getAll.foreach(println)
-    val ds = Seq(5, 6, 3, 8, 4)
-    val data: RDD[Int] = sc.parallelize(ds)
+    val seed: RDD[Int] = sc.parallelize(
+        Seq.fill(defaultNumOfPartitions)(defaultNumOfRecordsPerPartition), defaultNumOfPartitions)
+    val data = seed flatMap (recnum => Seq.fill(recnum)(Random.nextInt)) cache //must
be cached to fix rand numbers
     val durdd = data.makeDurable[Long](
         defaultServiceName,
         Array(DurableType.LONG), Array(),


Mime
View raw message